1 /* CPU control. 2 * (C) 2001, 2002, 2003, 2004 Rusty Russell 3 * 4 * This code is licenced under the GPL. 5 */ 6 #include <linux/proc_fs.h> 7 #include <linux/smp.h> 8 #include <linux/init.h> 9 #include <linux/notifier.h> 10 #include <linux/sched/signal.h> 11 #include <linux/sched/hotplug.h> 12 #include <linux/sched/task.h> 13 #include <linux/unistd.h> 14 #include <linux/cpu.h> 15 #include <linux/oom.h> 16 #include <linux/rcupdate.h> 17 #include <linux/export.h> 18 #include <linux/bug.h> 19 #include <linux/kthread.h> 20 #include <linux/stop_machine.h> 21 #include <linux/mutex.h> 22 #include <linux/gfp.h> 23 #include <linux/suspend.h> 24 #include <linux/lockdep.h> 25 #include <linux/tick.h> 26 #include <linux/irq.h> 27 #include <linux/smpboot.h> 28 #include <linux/relay.h> 29 #include <linux/slab.h> 30 #include <linux/percpu-rwsem.h> 31 32 #include <trace/events/power.h> 33 #define CREATE_TRACE_POINTS 34 #include <trace/events/cpuhp.h> 35 36 #include "smpboot.h" 37 38 /** 39 * cpuhp_cpu_state - Per cpu hotplug state storage 40 * @state: The current cpu state 41 * @target: The target state 42 * @thread: Pointer to the hotplug thread 43 * @should_run: Thread should execute 44 * @rollback: Perform a rollback 45 * @single: Single callback invocation 46 * @bringup: Single callback bringup or teardown selector 47 * @cb_state: The state for a single callback (install/uninstall) 48 * @result: Result of the operation 49 * @done: Signal completion to the issuer of the task 50 */ 51 struct cpuhp_cpu_state { 52 enum cpuhp_state state; 53 enum cpuhp_state target; 54 #ifdef CONFIG_SMP 55 struct task_struct *thread; 56 bool should_run; 57 bool rollback; 58 bool single; 59 bool bringup; 60 struct hlist_node *node; 61 enum cpuhp_state cb_state; 62 int result; 63 struct completion done; 64 #endif 65 }; 66 67 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state); 68 69 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP) 70 static struct lock_class_key cpuhp_state_key; 71 static struct lockdep_map cpuhp_state_lock_map = 72 STATIC_LOCKDEP_MAP_INIT("cpuhp_state", &cpuhp_state_key); 73 #endif 74 75 /** 76 * cpuhp_step - Hotplug state machine step 77 * @name: Name of the step 78 * @startup: Startup function of the step 79 * @teardown: Teardown function of the step 80 * @skip_onerr: Do not invoke the functions on error rollback 81 * Will go away once the notifiers are gone 82 * @cant_stop: Bringup/teardown can't be stopped at this step 83 */ 84 struct cpuhp_step { 85 const char *name; 86 union { 87 int (*single)(unsigned int cpu); 88 int (*multi)(unsigned int cpu, 89 struct hlist_node *node); 90 } startup; 91 union { 92 int (*single)(unsigned int cpu); 93 int (*multi)(unsigned int cpu, 94 struct hlist_node *node); 95 } teardown; 96 struct hlist_head list; 97 bool skip_onerr; 98 bool cant_stop; 99 bool multi_instance; 100 }; 101 102 static DEFINE_MUTEX(cpuhp_state_mutex); 103 static struct cpuhp_step cpuhp_bp_states[]; 104 static struct cpuhp_step cpuhp_ap_states[]; 105 106 static bool cpuhp_is_ap_state(enum cpuhp_state state) 107 { 108 /* 109 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation 110 * purposes as that state is handled explicitly in cpu_down. 111 */ 112 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU; 113 } 114 115 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state) 116 { 117 struct cpuhp_step *sp; 118 119 sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states; 120 return sp + state; 121 } 122 123 /** 124 * cpuhp_invoke_callback _ Invoke the callbacks for a given state 125 * @cpu: The cpu for which the callback should be invoked 126 * @step: The step in the state machine 127 * @bringup: True if the bringup callback should be invoked 128 * 129 * Called from cpu hotplug and from the state register machinery. 130 */ 131 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, 132 bool bringup, struct hlist_node *node) 133 { 134 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 135 struct cpuhp_step *step = cpuhp_get_step(state); 136 int (*cbm)(unsigned int cpu, struct hlist_node *node); 137 int (*cb)(unsigned int cpu); 138 int ret, cnt; 139 140 if (!step->multi_instance) { 141 cb = bringup ? step->startup.single : step->teardown.single; 142 if (!cb) 143 return 0; 144 trace_cpuhp_enter(cpu, st->target, state, cb); 145 ret = cb(cpu); 146 trace_cpuhp_exit(cpu, st->state, state, ret); 147 return ret; 148 } 149 cbm = bringup ? step->startup.multi : step->teardown.multi; 150 if (!cbm) 151 return 0; 152 153 /* Single invocation for instance add/remove */ 154 if (node) { 155 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); 156 ret = cbm(cpu, node); 157 trace_cpuhp_exit(cpu, st->state, state, ret); 158 return ret; 159 } 160 161 /* State transition. Invoke on all instances */ 162 cnt = 0; 163 hlist_for_each(node, &step->list) { 164 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); 165 ret = cbm(cpu, node); 166 trace_cpuhp_exit(cpu, st->state, state, ret); 167 if (ret) 168 goto err; 169 cnt++; 170 } 171 return 0; 172 err: 173 /* Rollback the instances if one failed */ 174 cbm = !bringup ? step->startup.multi : step->teardown.multi; 175 if (!cbm) 176 return ret; 177 178 hlist_for_each(node, &step->list) { 179 if (!cnt--) 180 break; 181 cbm(cpu, node); 182 } 183 return ret; 184 } 185 186 #ifdef CONFIG_SMP 187 /* Serializes the updates to cpu_online_mask, cpu_present_mask */ 188 static DEFINE_MUTEX(cpu_add_remove_lock); 189 bool cpuhp_tasks_frozen; 190 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen); 191 192 /* 193 * The following two APIs (cpu_maps_update_begin/done) must be used when 194 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. 195 */ 196 void cpu_maps_update_begin(void) 197 { 198 mutex_lock(&cpu_add_remove_lock); 199 } 200 201 void cpu_maps_update_done(void) 202 { 203 mutex_unlock(&cpu_add_remove_lock); 204 } 205 206 /* 207 * If set, cpu_up and cpu_down will return -EBUSY and do nothing. 208 * Should always be manipulated under cpu_add_remove_lock 209 */ 210 static int cpu_hotplug_disabled; 211 212 #ifdef CONFIG_HOTPLUG_CPU 213 214 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock); 215 216 void cpus_read_lock(void) 217 { 218 percpu_down_read(&cpu_hotplug_lock); 219 } 220 EXPORT_SYMBOL_GPL(cpus_read_lock); 221 222 void cpus_read_unlock(void) 223 { 224 percpu_up_read(&cpu_hotplug_lock); 225 } 226 EXPORT_SYMBOL_GPL(cpus_read_unlock); 227 228 void cpus_write_lock(void) 229 { 230 percpu_down_write(&cpu_hotplug_lock); 231 } 232 233 void cpus_write_unlock(void) 234 { 235 percpu_up_write(&cpu_hotplug_lock); 236 } 237 238 void lockdep_assert_cpus_held(void) 239 { 240 percpu_rwsem_assert_held(&cpu_hotplug_lock); 241 } 242 243 /* 244 * Wait for currently running CPU hotplug operations to complete (if any) and 245 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects 246 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the 247 * hotplug path before performing hotplug operations. So acquiring that lock 248 * guarantees mutual exclusion from any currently running hotplug operations. 249 */ 250 void cpu_hotplug_disable(void) 251 { 252 cpu_maps_update_begin(); 253 cpu_hotplug_disabled++; 254 cpu_maps_update_done(); 255 } 256 EXPORT_SYMBOL_GPL(cpu_hotplug_disable); 257 258 static void __cpu_hotplug_enable(void) 259 { 260 if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n")) 261 return; 262 cpu_hotplug_disabled--; 263 } 264 265 void cpu_hotplug_enable(void) 266 { 267 cpu_maps_update_begin(); 268 __cpu_hotplug_enable(); 269 cpu_maps_update_done(); 270 } 271 EXPORT_SYMBOL_GPL(cpu_hotplug_enable); 272 #endif /* CONFIG_HOTPLUG_CPU */ 273 274 static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st); 275 276 static int bringup_wait_for_ap(unsigned int cpu) 277 { 278 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 279 280 /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */ 281 wait_for_completion(&st->done); 282 BUG_ON(!cpu_online(cpu)); 283 284 /* Unpark the stopper thread and the hotplug thread of the target cpu */ 285 stop_machine_unpark(cpu); 286 kthread_unpark(st->thread); 287 288 /* Should we go further up ? */ 289 if (st->target > CPUHP_AP_ONLINE_IDLE) { 290 __cpuhp_kick_ap_work(st); 291 wait_for_completion(&st->done); 292 } 293 return st->result; 294 } 295 296 static int bringup_cpu(unsigned int cpu) 297 { 298 struct task_struct *idle = idle_thread_get(cpu); 299 int ret; 300 301 /* 302 * Some architectures have to walk the irq descriptors to 303 * setup the vector space for the cpu which comes online. 304 * Prevent irq alloc/free across the bringup. 305 */ 306 irq_lock_sparse(); 307 308 /* Arch-specific enabling code. */ 309 ret = __cpu_up(cpu, idle); 310 irq_unlock_sparse(); 311 if (ret) 312 return ret; 313 return bringup_wait_for_ap(cpu); 314 } 315 316 /* 317 * Hotplug state machine related functions 318 */ 319 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st) 320 { 321 for (st->state++; st->state < st->target; st->state++) { 322 struct cpuhp_step *step = cpuhp_get_step(st->state); 323 324 if (!step->skip_onerr) 325 cpuhp_invoke_callback(cpu, st->state, true, NULL); 326 } 327 } 328 329 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, 330 enum cpuhp_state target) 331 { 332 enum cpuhp_state prev_state = st->state; 333 int ret = 0; 334 335 for (; st->state > target; st->state--) { 336 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL); 337 if (ret) { 338 st->target = prev_state; 339 undo_cpu_down(cpu, st); 340 break; 341 } 342 } 343 return ret; 344 } 345 346 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st) 347 { 348 for (st->state--; st->state > st->target; st->state--) { 349 struct cpuhp_step *step = cpuhp_get_step(st->state); 350 351 if (!step->skip_onerr) 352 cpuhp_invoke_callback(cpu, st->state, false, NULL); 353 } 354 } 355 356 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, 357 enum cpuhp_state target) 358 { 359 enum cpuhp_state prev_state = st->state; 360 int ret = 0; 361 362 while (st->state < target) { 363 st->state++; 364 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL); 365 if (ret) { 366 st->target = prev_state; 367 undo_cpu_up(cpu, st); 368 break; 369 } 370 } 371 return ret; 372 } 373 374 /* 375 * The cpu hotplug threads manage the bringup and teardown of the cpus 376 */ 377 static void cpuhp_create(unsigned int cpu) 378 { 379 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 380 381 init_completion(&st->done); 382 } 383 384 static int cpuhp_should_run(unsigned int cpu) 385 { 386 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 387 388 return st->should_run; 389 } 390 391 /* Execute the teardown callbacks. Used to be CPU_DOWN_PREPARE */ 392 static int cpuhp_ap_offline(unsigned int cpu, struct cpuhp_cpu_state *st) 393 { 394 enum cpuhp_state target = max((int)st->target, CPUHP_TEARDOWN_CPU); 395 396 return cpuhp_down_callbacks(cpu, st, target); 397 } 398 399 /* Execute the online startup callbacks. Used to be CPU_ONLINE */ 400 static int cpuhp_ap_online(unsigned int cpu, struct cpuhp_cpu_state *st) 401 { 402 return cpuhp_up_callbacks(cpu, st, st->target); 403 } 404 405 /* 406 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke 407 * callbacks when a state gets [un]installed at runtime. 408 */ 409 static void cpuhp_thread_fun(unsigned int cpu) 410 { 411 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 412 int ret = 0; 413 414 /* 415 * Paired with the mb() in cpuhp_kick_ap_work and 416 * cpuhp_invoke_ap_callback, so the work set is consistent visible. 417 */ 418 smp_mb(); 419 if (!st->should_run) 420 return; 421 422 st->should_run = false; 423 424 lock_map_acquire(&cpuhp_state_lock_map); 425 /* Single callback invocation for [un]install ? */ 426 if (st->single) { 427 if (st->cb_state < CPUHP_AP_ONLINE) { 428 local_irq_disable(); 429 ret = cpuhp_invoke_callback(cpu, st->cb_state, 430 st->bringup, st->node); 431 local_irq_enable(); 432 } else { 433 ret = cpuhp_invoke_callback(cpu, st->cb_state, 434 st->bringup, st->node); 435 } 436 } else if (st->rollback) { 437 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE); 438 439 undo_cpu_down(cpu, st); 440 st->rollback = false; 441 } else { 442 /* Cannot happen .... */ 443 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE); 444 445 /* Regular hotplug work */ 446 if (st->state < st->target) 447 ret = cpuhp_ap_online(cpu, st); 448 else if (st->state > st->target) 449 ret = cpuhp_ap_offline(cpu, st); 450 } 451 lock_map_release(&cpuhp_state_lock_map); 452 st->result = ret; 453 complete(&st->done); 454 } 455 456 /* Invoke a single callback on a remote cpu */ 457 static int 458 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup, 459 struct hlist_node *node) 460 { 461 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 462 463 if (!cpu_online(cpu)) 464 return 0; 465 466 lock_map_acquire(&cpuhp_state_lock_map); 467 lock_map_release(&cpuhp_state_lock_map); 468 469 /* 470 * If we are up and running, use the hotplug thread. For early calls 471 * we invoke the thread function directly. 472 */ 473 if (!st->thread) 474 return cpuhp_invoke_callback(cpu, state, bringup, node); 475 476 st->cb_state = state; 477 st->single = true; 478 st->bringup = bringup; 479 st->node = node; 480 481 /* 482 * Make sure the above stores are visible before should_run becomes 483 * true. Paired with the mb() above in cpuhp_thread_fun() 484 */ 485 smp_mb(); 486 st->should_run = true; 487 wake_up_process(st->thread); 488 wait_for_completion(&st->done); 489 return st->result; 490 } 491 492 /* Regular hotplug invocation of the AP hotplug thread */ 493 static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st) 494 { 495 st->result = 0; 496 st->single = false; 497 /* 498 * Make sure the above stores are visible before should_run becomes 499 * true. Paired with the mb() above in cpuhp_thread_fun() 500 */ 501 smp_mb(); 502 st->should_run = true; 503 wake_up_process(st->thread); 504 } 505 506 static int cpuhp_kick_ap_work(unsigned int cpu) 507 { 508 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 509 enum cpuhp_state state = st->state; 510 511 trace_cpuhp_enter(cpu, st->target, state, cpuhp_kick_ap_work); 512 lock_map_acquire(&cpuhp_state_lock_map); 513 lock_map_release(&cpuhp_state_lock_map); 514 __cpuhp_kick_ap_work(st); 515 wait_for_completion(&st->done); 516 trace_cpuhp_exit(cpu, st->state, state, st->result); 517 return st->result; 518 } 519 520 static struct smp_hotplug_thread cpuhp_threads = { 521 .store = &cpuhp_state.thread, 522 .create = &cpuhp_create, 523 .thread_should_run = cpuhp_should_run, 524 .thread_fn = cpuhp_thread_fun, 525 .thread_comm = "cpuhp/%u", 526 .selfparking = true, 527 }; 528 529 void __init cpuhp_threads_init(void) 530 { 531 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads)); 532 kthread_unpark(this_cpu_read(cpuhp_state.thread)); 533 } 534 535 #ifdef CONFIG_HOTPLUG_CPU 536 /** 537 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU 538 * @cpu: a CPU id 539 * 540 * This function walks all processes, finds a valid mm struct for each one and 541 * then clears a corresponding bit in mm's cpumask. While this all sounds 542 * trivial, there are various non-obvious corner cases, which this function 543 * tries to solve in a safe manner. 544 * 545 * Also note that the function uses a somewhat relaxed locking scheme, so it may 546 * be called only for an already offlined CPU. 547 */ 548 void clear_tasks_mm_cpumask(int cpu) 549 { 550 struct task_struct *p; 551 552 /* 553 * This function is called after the cpu is taken down and marked 554 * offline, so its not like new tasks will ever get this cpu set in 555 * their mm mask. -- Peter Zijlstra 556 * Thus, we may use rcu_read_lock() here, instead of grabbing 557 * full-fledged tasklist_lock. 558 */ 559 WARN_ON(cpu_online(cpu)); 560 rcu_read_lock(); 561 for_each_process(p) { 562 struct task_struct *t; 563 564 /* 565 * Main thread might exit, but other threads may still have 566 * a valid mm. Find one. 567 */ 568 t = find_lock_task_mm(p); 569 if (!t) 570 continue; 571 cpumask_clear_cpu(cpu, mm_cpumask(t->mm)); 572 task_unlock(t); 573 } 574 rcu_read_unlock(); 575 } 576 577 /* Take this CPU down. */ 578 static int take_cpu_down(void *_param) 579 { 580 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 581 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); 582 int err, cpu = smp_processor_id(); 583 584 /* Ensure this CPU doesn't handle any more interrupts. */ 585 err = __cpu_disable(); 586 if (err < 0) 587 return err; 588 589 /* 590 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not 591 * do this step again. 592 */ 593 WARN_ON(st->state != CPUHP_TEARDOWN_CPU); 594 st->state--; 595 /* Invoke the former CPU_DYING callbacks */ 596 for (; st->state > target; st->state--) 597 cpuhp_invoke_callback(cpu, st->state, false, NULL); 598 599 /* Give up timekeeping duties */ 600 tick_handover_do_timer(); 601 /* Park the stopper thread */ 602 stop_machine_park(cpu); 603 return 0; 604 } 605 606 static int takedown_cpu(unsigned int cpu) 607 { 608 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 609 int err; 610 611 /* Park the smpboot threads */ 612 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread); 613 smpboot_park_threads(cpu); 614 615 /* 616 * Prevent irq alloc/free while the dying cpu reorganizes the 617 * interrupt affinities. 618 */ 619 irq_lock_sparse(); 620 621 /* 622 * So now all preempt/rcu users must observe !cpu_active(). 623 */ 624 err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu)); 625 if (err) { 626 /* CPU refused to die */ 627 irq_unlock_sparse(); 628 /* Unpark the hotplug thread so we can rollback there */ 629 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread); 630 return err; 631 } 632 BUG_ON(cpu_online(cpu)); 633 634 /* 635 * The CPUHP_AP_SCHED_MIGRATE_DYING callback will have removed all 636 * runnable tasks from the cpu, there's only the idle task left now 637 * that the migration thread is done doing the stop_machine thing. 638 * 639 * Wait for the stop thread to go away. 640 */ 641 wait_for_completion(&st->done); 642 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); 643 644 /* Interrupts are moved away from the dying cpu, reenable alloc/free */ 645 irq_unlock_sparse(); 646 647 hotplug_cpu__broadcast_tick_pull(cpu); 648 /* This actually kills the CPU. */ 649 __cpu_die(cpu); 650 651 tick_cleanup_dead_cpu(cpu); 652 return 0; 653 } 654 655 static void cpuhp_complete_idle_dead(void *arg) 656 { 657 struct cpuhp_cpu_state *st = arg; 658 659 complete(&st->done); 660 } 661 662 void cpuhp_report_idle_dead(void) 663 { 664 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 665 666 BUG_ON(st->state != CPUHP_AP_OFFLINE); 667 rcu_report_dead(smp_processor_id()); 668 st->state = CPUHP_AP_IDLE_DEAD; 669 /* 670 * We cannot call complete after rcu_report_dead() so we delegate it 671 * to an online cpu. 672 */ 673 smp_call_function_single(cpumask_first(cpu_online_mask), 674 cpuhp_complete_idle_dead, st, 0); 675 } 676 677 #else 678 #define takedown_cpu NULL 679 #endif 680 681 #ifdef CONFIG_HOTPLUG_CPU 682 683 /* Requires cpu_add_remove_lock to be held */ 684 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, 685 enum cpuhp_state target) 686 { 687 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 688 int prev_state, ret = 0; 689 690 if (num_online_cpus() == 1) 691 return -EBUSY; 692 693 if (!cpu_present(cpu)) 694 return -EINVAL; 695 696 cpus_write_lock(); 697 698 cpuhp_tasks_frozen = tasks_frozen; 699 700 prev_state = st->state; 701 st->target = target; 702 /* 703 * If the current CPU state is in the range of the AP hotplug thread, 704 * then we need to kick the thread. 705 */ 706 if (st->state > CPUHP_TEARDOWN_CPU) { 707 ret = cpuhp_kick_ap_work(cpu); 708 /* 709 * The AP side has done the error rollback already. Just 710 * return the error code.. 711 */ 712 if (ret) 713 goto out; 714 715 /* 716 * We might have stopped still in the range of the AP hotplug 717 * thread. Nothing to do anymore. 718 */ 719 if (st->state > CPUHP_TEARDOWN_CPU) 720 goto out; 721 } 722 /* 723 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need 724 * to do the further cleanups. 725 */ 726 ret = cpuhp_down_callbacks(cpu, st, target); 727 if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) { 728 st->target = prev_state; 729 st->rollback = true; 730 cpuhp_kick_ap_work(cpu); 731 } 732 733 out: 734 cpus_write_unlock(); 735 return ret; 736 } 737 738 static int do_cpu_down(unsigned int cpu, enum cpuhp_state target) 739 { 740 int err; 741 742 cpu_maps_update_begin(); 743 744 if (cpu_hotplug_disabled) { 745 err = -EBUSY; 746 goto out; 747 } 748 749 err = _cpu_down(cpu, 0, target); 750 751 out: 752 cpu_maps_update_done(); 753 return err; 754 } 755 int cpu_down(unsigned int cpu) 756 { 757 return do_cpu_down(cpu, CPUHP_OFFLINE); 758 } 759 EXPORT_SYMBOL(cpu_down); 760 #endif /*CONFIG_HOTPLUG_CPU*/ 761 762 /** 763 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU 764 * @cpu: cpu that just started 765 * 766 * It must be called by the arch code on the new cpu, before the new cpu 767 * enables interrupts and before the "boot" cpu returns from __cpu_up(). 768 */ 769 void notify_cpu_starting(unsigned int cpu) 770 { 771 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 772 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); 773 774 rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */ 775 while (st->state < target) { 776 st->state++; 777 cpuhp_invoke_callback(cpu, st->state, true, NULL); 778 } 779 } 780 781 /* 782 * Called from the idle task. Wake up the controlling task which brings the 783 * stopper and the hotplug thread of the upcoming CPU up and then delegates 784 * the rest of the online bringup to the hotplug thread. 785 */ 786 void cpuhp_online_idle(enum cpuhp_state state) 787 { 788 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 789 790 /* Happens for the boot cpu */ 791 if (state != CPUHP_AP_ONLINE_IDLE) 792 return; 793 794 st->state = CPUHP_AP_ONLINE_IDLE; 795 complete(&st->done); 796 } 797 798 /* Requires cpu_add_remove_lock to be held */ 799 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target) 800 { 801 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 802 struct task_struct *idle; 803 int ret = 0; 804 805 cpus_write_lock(); 806 807 if (!cpu_present(cpu)) { 808 ret = -EINVAL; 809 goto out; 810 } 811 812 /* 813 * The caller of do_cpu_up might have raced with another 814 * caller. Ignore it for now. 815 */ 816 if (st->state >= target) 817 goto out; 818 819 if (st->state == CPUHP_OFFLINE) { 820 /* Let it fail before we try to bring the cpu up */ 821 idle = idle_thread_get(cpu); 822 if (IS_ERR(idle)) { 823 ret = PTR_ERR(idle); 824 goto out; 825 } 826 } 827 828 cpuhp_tasks_frozen = tasks_frozen; 829 830 st->target = target; 831 /* 832 * If the current CPU state is in the range of the AP hotplug thread, 833 * then we need to kick the thread once more. 834 */ 835 if (st->state > CPUHP_BRINGUP_CPU) { 836 ret = cpuhp_kick_ap_work(cpu); 837 /* 838 * The AP side has done the error rollback already. Just 839 * return the error code.. 840 */ 841 if (ret) 842 goto out; 843 } 844 845 /* 846 * Try to reach the target state. We max out on the BP at 847 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is 848 * responsible for bringing it up to the target state. 849 */ 850 target = min((int)target, CPUHP_BRINGUP_CPU); 851 ret = cpuhp_up_callbacks(cpu, st, target); 852 out: 853 cpus_write_unlock(); 854 return ret; 855 } 856 857 static int do_cpu_up(unsigned int cpu, enum cpuhp_state target) 858 { 859 int err = 0; 860 861 if (!cpu_possible(cpu)) { 862 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n", 863 cpu); 864 #if defined(CONFIG_IA64) 865 pr_err("please check additional_cpus= boot parameter\n"); 866 #endif 867 return -EINVAL; 868 } 869 870 err = try_online_node(cpu_to_node(cpu)); 871 if (err) 872 return err; 873 874 cpu_maps_update_begin(); 875 876 if (cpu_hotplug_disabled) { 877 err = -EBUSY; 878 goto out; 879 } 880 881 err = _cpu_up(cpu, 0, target); 882 out: 883 cpu_maps_update_done(); 884 return err; 885 } 886 887 int cpu_up(unsigned int cpu) 888 { 889 return do_cpu_up(cpu, CPUHP_ONLINE); 890 } 891 EXPORT_SYMBOL_GPL(cpu_up); 892 893 #ifdef CONFIG_PM_SLEEP_SMP 894 static cpumask_var_t frozen_cpus; 895 896 int freeze_secondary_cpus(int primary) 897 { 898 int cpu, error = 0; 899 900 cpu_maps_update_begin(); 901 if (!cpu_online(primary)) 902 primary = cpumask_first(cpu_online_mask); 903 /* 904 * We take down all of the non-boot CPUs in one shot to avoid races 905 * with the userspace trying to use the CPU hotplug at the same time 906 */ 907 cpumask_clear(frozen_cpus); 908 909 pr_info("Disabling non-boot CPUs ...\n"); 910 for_each_online_cpu(cpu) { 911 if (cpu == primary) 912 continue; 913 trace_suspend_resume(TPS("CPU_OFF"), cpu, true); 914 error = _cpu_down(cpu, 1, CPUHP_OFFLINE); 915 trace_suspend_resume(TPS("CPU_OFF"), cpu, false); 916 if (!error) 917 cpumask_set_cpu(cpu, frozen_cpus); 918 else { 919 pr_err("Error taking CPU%d down: %d\n", cpu, error); 920 break; 921 } 922 } 923 924 if (!error) 925 BUG_ON(num_online_cpus() > 1); 926 else 927 pr_err("Non-boot CPUs are not disabled\n"); 928 929 /* 930 * Make sure the CPUs won't be enabled by someone else. We need to do 931 * this even in case of failure as all disable_nonboot_cpus() users are 932 * supposed to do enable_nonboot_cpus() on the failure path. 933 */ 934 cpu_hotplug_disabled++; 935 936 cpu_maps_update_done(); 937 return error; 938 } 939 940 void __weak arch_enable_nonboot_cpus_begin(void) 941 { 942 } 943 944 void __weak arch_enable_nonboot_cpus_end(void) 945 { 946 } 947 948 void enable_nonboot_cpus(void) 949 { 950 int cpu, error; 951 952 /* Allow everyone to use the CPU hotplug again */ 953 cpu_maps_update_begin(); 954 __cpu_hotplug_enable(); 955 if (cpumask_empty(frozen_cpus)) 956 goto out; 957 958 pr_info("Enabling non-boot CPUs ...\n"); 959 960 arch_enable_nonboot_cpus_begin(); 961 962 for_each_cpu(cpu, frozen_cpus) { 963 trace_suspend_resume(TPS("CPU_ON"), cpu, true); 964 error = _cpu_up(cpu, 1, CPUHP_ONLINE); 965 trace_suspend_resume(TPS("CPU_ON"), cpu, false); 966 if (!error) { 967 pr_info("CPU%d is up\n", cpu); 968 continue; 969 } 970 pr_warn("Error taking CPU%d up: %d\n", cpu, error); 971 } 972 973 arch_enable_nonboot_cpus_end(); 974 975 cpumask_clear(frozen_cpus); 976 out: 977 cpu_maps_update_done(); 978 } 979 980 static int __init alloc_frozen_cpus(void) 981 { 982 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO)) 983 return -ENOMEM; 984 return 0; 985 } 986 core_initcall(alloc_frozen_cpus); 987 988 /* 989 * When callbacks for CPU hotplug notifications are being executed, we must 990 * ensure that the state of the system with respect to the tasks being frozen 991 * or not, as reported by the notification, remains unchanged *throughout the 992 * duration* of the execution of the callbacks. 993 * Hence we need to prevent the freezer from racing with regular CPU hotplug. 994 * 995 * This synchronization is implemented by mutually excluding regular CPU 996 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/ 997 * Hibernate notifications. 998 */ 999 static int 1000 cpu_hotplug_pm_callback(struct notifier_block *nb, 1001 unsigned long action, void *ptr) 1002 { 1003 switch (action) { 1004 1005 case PM_SUSPEND_PREPARE: 1006 case PM_HIBERNATION_PREPARE: 1007 cpu_hotplug_disable(); 1008 break; 1009 1010 case PM_POST_SUSPEND: 1011 case PM_POST_HIBERNATION: 1012 cpu_hotplug_enable(); 1013 break; 1014 1015 default: 1016 return NOTIFY_DONE; 1017 } 1018 1019 return NOTIFY_OK; 1020 } 1021 1022 1023 static int __init cpu_hotplug_pm_sync_init(void) 1024 { 1025 /* 1026 * cpu_hotplug_pm_callback has higher priority than x86 1027 * bsp_pm_callback which depends on cpu_hotplug_pm_callback 1028 * to disable cpu hotplug to avoid cpu hotplug race. 1029 */ 1030 pm_notifier(cpu_hotplug_pm_callback, 0); 1031 return 0; 1032 } 1033 core_initcall(cpu_hotplug_pm_sync_init); 1034 1035 #endif /* CONFIG_PM_SLEEP_SMP */ 1036 1037 int __boot_cpu_id; 1038 1039 #endif /* CONFIG_SMP */ 1040 1041 /* Boot processor state steps */ 1042 static struct cpuhp_step cpuhp_bp_states[] = { 1043 [CPUHP_OFFLINE] = { 1044 .name = "offline", 1045 .startup.single = NULL, 1046 .teardown.single = NULL, 1047 }, 1048 #ifdef CONFIG_SMP 1049 [CPUHP_CREATE_THREADS]= { 1050 .name = "threads:prepare", 1051 .startup.single = smpboot_create_threads, 1052 .teardown.single = NULL, 1053 .cant_stop = true, 1054 }, 1055 [CPUHP_PERF_PREPARE] = { 1056 .name = "perf:prepare", 1057 .startup.single = perf_event_init_cpu, 1058 .teardown.single = perf_event_exit_cpu, 1059 }, 1060 [CPUHP_WORKQUEUE_PREP] = { 1061 .name = "workqueue:prepare", 1062 .startup.single = workqueue_prepare_cpu, 1063 .teardown.single = NULL, 1064 }, 1065 [CPUHP_HRTIMERS_PREPARE] = { 1066 .name = "hrtimers:prepare", 1067 .startup.single = hrtimers_prepare_cpu, 1068 .teardown.single = hrtimers_dead_cpu, 1069 }, 1070 [CPUHP_SMPCFD_PREPARE] = { 1071 .name = "smpcfd:prepare", 1072 .startup.single = smpcfd_prepare_cpu, 1073 .teardown.single = smpcfd_dead_cpu, 1074 }, 1075 [CPUHP_RELAY_PREPARE] = { 1076 .name = "relay:prepare", 1077 .startup.single = relay_prepare_cpu, 1078 .teardown.single = NULL, 1079 }, 1080 [CPUHP_SLAB_PREPARE] = { 1081 .name = "slab:prepare", 1082 .startup.single = slab_prepare_cpu, 1083 .teardown.single = slab_dead_cpu, 1084 }, 1085 [CPUHP_RCUTREE_PREP] = { 1086 .name = "RCU/tree:prepare", 1087 .startup.single = rcutree_prepare_cpu, 1088 .teardown.single = rcutree_dead_cpu, 1089 }, 1090 /* 1091 * On the tear-down path, timers_dead_cpu() must be invoked 1092 * before blk_mq_queue_reinit_notify() from notify_dead(), 1093 * otherwise a RCU stall occurs. 1094 */ 1095 [CPUHP_TIMERS_DEAD] = { 1096 .name = "timers:dead", 1097 .startup.single = NULL, 1098 .teardown.single = timers_dead_cpu, 1099 }, 1100 /* Kicks the plugged cpu into life */ 1101 [CPUHP_BRINGUP_CPU] = { 1102 .name = "cpu:bringup", 1103 .startup.single = bringup_cpu, 1104 .teardown.single = NULL, 1105 .cant_stop = true, 1106 }, 1107 [CPUHP_AP_SMPCFD_DYING] = { 1108 .name = "smpcfd:dying", 1109 .startup.single = NULL, 1110 .teardown.single = smpcfd_dying_cpu, 1111 }, 1112 /* 1113 * Handled on controll processor until the plugged processor manages 1114 * this itself. 1115 */ 1116 [CPUHP_TEARDOWN_CPU] = { 1117 .name = "cpu:teardown", 1118 .startup.single = NULL, 1119 .teardown.single = takedown_cpu, 1120 .cant_stop = true, 1121 }, 1122 #else 1123 [CPUHP_BRINGUP_CPU] = { }, 1124 #endif 1125 }; 1126 1127 /* Application processor state steps */ 1128 static struct cpuhp_step cpuhp_ap_states[] = { 1129 #ifdef CONFIG_SMP 1130 /* Final state before CPU kills itself */ 1131 [CPUHP_AP_IDLE_DEAD] = { 1132 .name = "idle:dead", 1133 }, 1134 /* 1135 * Last state before CPU enters the idle loop to die. Transient state 1136 * for synchronization. 1137 */ 1138 [CPUHP_AP_OFFLINE] = { 1139 .name = "ap:offline", 1140 .cant_stop = true, 1141 }, 1142 /* First state is scheduler control. Interrupts are disabled */ 1143 [CPUHP_AP_SCHED_STARTING] = { 1144 .name = "sched:starting", 1145 .startup.single = sched_cpu_starting, 1146 .teardown.single = sched_cpu_dying, 1147 }, 1148 [CPUHP_AP_RCUTREE_DYING] = { 1149 .name = "RCU/tree:dying", 1150 .startup.single = NULL, 1151 .teardown.single = rcutree_dying_cpu, 1152 }, 1153 /* Entry state on starting. Interrupts enabled from here on. Transient 1154 * state for synchronsization */ 1155 [CPUHP_AP_ONLINE] = { 1156 .name = "ap:online", 1157 }, 1158 /* Handle smpboot threads park/unpark */ 1159 [CPUHP_AP_SMPBOOT_THREADS] = { 1160 .name = "smpboot/threads:online", 1161 .startup.single = smpboot_unpark_threads, 1162 .teardown.single = NULL, 1163 }, 1164 [CPUHP_AP_IRQ_AFFINITY_ONLINE] = { 1165 .name = "irq/affinity:online", 1166 .startup.single = irq_affinity_online_cpu, 1167 .teardown.single = NULL, 1168 }, 1169 [CPUHP_AP_PERF_ONLINE] = { 1170 .name = "perf:online", 1171 .startup.single = perf_event_init_cpu, 1172 .teardown.single = perf_event_exit_cpu, 1173 }, 1174 [CPUHP_AP_WORKQUEUE_ONLINE] = { 1175 .name = "workqueue:online", 1176 .startup.single = workqueue_online_cpu, 1177 .teardown.single = workqueue_offline_cpu, 1178 }, 1179 [CPUHP_AP_RCUTREE_ONLINE] = { 1180 .name = "RCU/tree:online", 1181 .startup.single = rcutree_online_cpu, 1182 .teardown.single = rcutree_offline_cpu, 1183 }, 1184 #endif 1185 /* 1186 * The dynamically registered state space is here 1187 */ 1188 1189 #ifdef CONFIG_SMP 1190 /* Last state is scheduler control setting the cpu active */ 1191 [CPUHP_AP_ACTIVE] = { 1192 .name = "sched:active", 1193 .startup.single = sched_cpu_activate, 1194 .teardown.single = sched_cpu_deactivate, 1195 }, 1196 #endif 1197 1198 /* CPU is fully up and running. */ 1199 [CPUHP_ONLINE] = { 1200 .name = "online", 1201 .startup.single = NULL, 1202 .teardown.single = NULL, 1203 }, 1204 }; 1205 1206 /* Sanity check for callbacks */ 1207 static int cpuhp_cb_check(enum cpuhp_state state) 1208 { 1209 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE) 1210 return -EINVAL; 1211 return 0; 1212 } 1213 1214 /* 1215 * Returns a free for dynamic slot assignment of the Online state. The states 1216 * are protected by the cpuhp_slot_states mutex and an empty slot is identified 1217 * by having no name assigned. 1218 */ 1219 static int cpuhp_reserve_state(enum cpuhp_state state) 1220 { 1221 enum cpuhp_state i, end; 1222 struct cpuhp_step *step; 1223 1224 switch (state) { 1225 case CPUHP_AP_ONLINE_DYN: 1226 step = cpuhp_ap_states + CPUHP_AP_ONLINE_DYN; 1227 end = CPUHP_AP_ONLINE_DYN_END; 1228 break; 1229 case CPUHP_BP_PREPARE_DYN: 1230 step = cpuhp_bp_states + CPUHP_BP_PREPARE_DYN; 1231 end = CPUHP_BP_PREPARE_DYN_END; 1232 break; 1233 default: 1234 return -EINVAL; 1235 } 1236 1237 for (i = state; i <= end; i++, step++) { 1238 if (!step->name) 1239 return i; 1240 } 1241 WARN(1, "No more dynamic states available for CPU hotplug\n"); 1242 return -ENOSPC; 1243 } 1244 1245 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name, 1246 int (*startup)(unsigned int cpu), 1247 int (*teardown)(unsigned int cpu), 1248 bool multi_instance) 1249 { 1250 /* (Un)Install the callbacks for further cpu hotplug operations */ 1251 struct cpuhp_step *sp; 1252 int ret = 0; 1253 1254 if (state == CPUHP_AP_ONLINE_DYN || state == CPUHP_BP_PREPARE_DYN) { 1255 ret = cpuhp_reserve_state(state); 1256 if (ret < 0) 1257 return ret; 1258 state = ret; 1259 } 1260 sp = cpuhp_get_step(state); 1261 if (name && sp->name) 1262 return -EBUSY; 1263 1264 sp->startup.single = startup; 1265 sp->teardown.single = teardown; 1266 sp->name = name; 1267 sp->multi_instance = multi_instance; 1268 INIT_HLIST_HEAD(&sp->list); 1269 return ret; 1270 } 1271 1272 static void *cpuhp_get_teardown_cb(enum cpuhp_state state) 1273 { 1274 return cpuhp_get_step(state)->teardown.single; 1275 } 1276 1277 /* 1278 * Call the startup/teardown function for a step either on the AP or 1279 * on the current CPU. 1280 */ 1281 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup, 1282 struct hlist_node *node) 1283 { 1284 struct cpuhp_step *sp = cpuhp_get_step(state); 1285 int ret; 1286 1287 if ((bringup && !sp->startup.single) || 1288 (!bringup && !sp->teardown.single)) 1289 return 0; 1290 /* 1291 * The non AP bound callbacks can fail on bringup. On teardown 1292 * e.g. module removal we crash for now. 1293 */ 1294 #ifdef CONFIG_SMP 1295 if (cpuhp_is_ap_state(state)) 1296 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node); 1297 else 1298 ret = cpuhp_invoke_callback(cpu, state, bringup, node); 1299 #else 1300 ret = cpuhp_invoke_callback(cpu, state, bringup, node); 1301 #endif 1302 BUG_ON(ret && !bringup); 1303 return ret; 1304 } 1305 1306 /* 1307 * Called from __cpuhp_setup_state on a recoverable failure. 1308 * 1309 * Note: The teardown callbacks for rollback are not allowed to fail! 1310 */ 1311 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state, 1312 struct hlist_node *node) 1313 { 1314 int cpu; 1315 1316 /* Roll back the already executed steps on the other cpus */ 1317 for_each_present_cpu(cpu) { 1318 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1319 int cpustate = st->state; 1320 1321 if (cpu >= failedcpu) 1322 break; 1323 1324 /* Did we invoke the startup call on that cpu ? */ 1325 if (cpustate >= state) 1326 cpuhp_issue_call(cpu, state, false, node); 1327 } 1328 } 1329 1330 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state, 1331 struct hlist_node *node, 1332 bool invoke) 1333 { 1334 struct cpuhp_step *sp; 1335 int cpu; 1336 int ret; 1337 1338 lockdep_assert_cpus_held(); 1339 1340 sp = cpuhp_get_step(state); 1341 if (sp->multi_instance == false) 1342 return -EINVAL; 1343 1344 mutex_lock(&cpuhp_state_mutex); 1345 1346 if (!invoke || !sp->startup.multi) 1347 goto add_node; 1348 1349 /* 1350 * Try to call the startup callback for each present cpu 1351 * depending on the hotplug state of the cpu. 1352 */ 1353 for_each_present_cpu(cpu) { 1354 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1355 int cpustate = st->state; 1356 1357 if (cpustate < state) 1358 continue; 1359 1360 ret = cpuhp_issue_call(cpu, state, true, node); 1361 if (ret) { 1362 if (sp->teardown.multi) 1363 cpuhp_rollback_install(cpu, state, node); 1364 goto unlock; 1365 } 1366 } 1367 add_node: 1368 ret = 0; 1369 hlist_add_head(node, &sp->list); 1370 unlock: 1371 mutex_unlock(&cpuhp_state_mutex); 1372 return ret; 1373 } 1374 1375 int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node, 1376 bool invoke) 1377 { 1378 int ret; 1379 1380 cpus_read_lock(); 1381 ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke); 1382 cpus_read_unlock(); 1383 return ret; 1384 } 1385 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance); 1386 1387 /** 1388 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state 1389 * @state: The state to setup 1390 * @invoke: If true, the startup function is invoked for cpus where 1391 * cpu state >= @state 1392 * @startup: startup callback function 1393 * @teardown: teardown callback function 1394 * @multi_instance: State is set up for multiple instances which get 1395 * added afterwards. 1396 * 1397 * The caller needs to hold cpus read locked while calling this function. 1398 * Returns: 1399 * On success: 1400 * Positive state number if @state is CPUHP_AP_ONLINE_DYN 1401 * 0 for all other states 1402 * On failure: proper (negative) error code 1403 */ 1404 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state, 1405 const char *name, bool invoke, 1406 int (*startup)(unsigned int cpu), 1407 int (*teardown)(unsigned int cpu), 1408 bool multi_instance) 1409 { 1410 int cpu, ret = 0; 1411 bool dynstate; 1412 1413 lockdep_assert_cpus_held(); 1414 1415 if (cpuhp_cb_check(state) || !name) 1416 return -EINVAL; 1417 1418 mutex_lock(&cpuhp_state_mutex); 1419 1420 ret = cpuhp_store_callbacks(state, name, startup, teardown, 1421 multi_instance); 1422 1423 dynstate = state == CPUHP_AP_ONLINE_DYN; 1424 if (ret > 0 && dynstate) { 1425 state = ret; 1426 ret = 0; 1427 } 1428 1429 if (ret || !invoke || !startup) 1430 goto out; 1431 1432 /* 1433 * Try to call the startup callback for each present cpu 1434 * depending on the hotplug state of the cpu. 1435 */ 1436 for_each_present_cpu(cpu) { 1437 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1438 int cpustate = st->state; 1439 1440 if (cpustate < state) 1441 continue; 1442 1443 ret = cpuhp_issue_call(cpu, state, true, NULL); 1444 if (ret) { 1445 if (teardown) 1446 cpuhp_rollback_install(cpu, state, NULL); 1447 cpuhp_store_callbacks(state, NULL, NULL, NULL, false); 1448 goto out; 1449 } 1450 } 1451 out: 1452 mutex_unlock(&cpuhp_state_mutex); 1453 /* 1454 * If the requested state is CPUHP_AP_ONLINE_DYN, return the 1455 * dynamically allocated state in case of success. 1456 */ 1457 if (!ret && dynstate) 1458 return state; 1459 return ret; 1460 } 1461 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked); 1462 1463 int __cpuhp_setup_state(enum cpuhp_state state, 1464 const char *name, bool invoke, 1465 int (*startup)(unsigned int cpu), 1466 int (*teardown)(unsigned int cpu), 1467 bool multi_instance) 1468 { 1469 int ret; 1470 1471 cpus_read_lock(); 1472 ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup, 1473 teardown, multi_instance); 1474 cpus_read_unlock(); 1475 return ret; 1476 } 1477 EXPORT_SYMBOL(__cpuhp_setup_state); 1478 1479 int __cpuhp_state_remove_instance(enum cpuhp_state state, 1480 struct hlist_node *node, bool invoke) 1481 { 1482 struct cpuhp_step *sp = cpuhp_get_step(state); 1483 int cpu; 1484 1485 BUG_ON(cpuhp_cb_check(state)); 1486 1487 if (!sp->multi_instance) 1488 return -EINVAL; 1489 1490 cpus_read_lock(); 1491 mutex_lock(&cpuhp_state_mutex); 1492 1493 if (!invoke || !cpuhp_get_teardown_cb(state)) 1494 goto remove; 1495 /* 1496 * Call the teardown callback for each present cpu depending 1497 * on the hotplug state of the cpu. This function is not 1498 * allowed to fail currently! 1499 */ 1500 for_each_present_cpu(cpu) { 1501 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1502 int cpustate = st->state; 1503 1504 if (cpustate >= state) 1505 cpuhp_issue_call(cpu, state, false, node); 1506 } 1507 1508 remove: 1509 hlist_del(node); 1510 mutex_unlock(&cpuhp_state_mutex); 1511 cpus_read_unlock(); 1512 1513 return 0; 1514 } 1515 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance); 1516 1517 /** 1518 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state 1519 * @state: The state to remove 1520 * @invoke: If true, the teardown function is invoked for cpus where 1521 * cpu state >= @state 1522 * 1523 * The caller needs to hold cpus read locked while calling this function. 1524 * The teardown callback is currently not allowed to fail. Think 1525 * about module removal! 1526 */ 1527 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke) 1528 { 1529 struct cpuhp_step *sp = cpuhp_get_step(state); 1530 int cpu; 1531 1532 BUG_ON(cpuhp_cb_check(state)); 1533 1534 lockdep_assert_cpus_held(); 1535 1536 mutex_lock(&cpuhp_state_mutex); 1537 if (sp->multi_instance) { 1538 WARN(!hlist_empty(&sp->list), 1539 "Error: Removing state %d which has instances left.\n", 1540 state); 1541 goto remove; 1542 } 1543 1544 if (!invoke || !cpuhp_get_teardown_cb(state)) 1545 goto remove; 1546 1547 /* 1548 * Call the teardown callback for each present cpu depending 1549 * on the hotplug state of the cpu. This function is not 1550 * allowed to fail currently! 1551 */ 1552 for_each_present_cpu(cpu) { 1553 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1554 int cpustate = st->state; 1555 1556 if (cpustate >= state) 1557 cpuhp_issue_call(cpu, state, false, NULL); 1558 } 1559 remove: 1560 cpuhp_store_callbacks(state, NULL, NULL, NULL, false); 1561 mutex_unlock(&cpuhp_state_mutex); 1562 } 1563 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked); 1564 1565 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke) 1566 { 1567 cpus_read_lock(); 1568 __cpuhp_remove_state_cpuslocked(state, invoke); 1569 cpus_read_unlock(); 1570 } 1571 EXPORT_SYMBOL(__cpuhp_remove_state); 1572 1573 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU) 1574 static ssize_t show_cpuhp_state(struct device *dev, 1575 struct device_attribute *attr, char *buf) 1576 { 1577 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1578 1579 return sprintf(buf, "%d\n", st->state); 1580 } 1581 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL); 1582 1583 static ssize_t write_cpuhp_target(struct device *dev, 1584 struct device_attribute *attr, 1585 const char *buf, size_t count) 1586 { 1587 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1588 struct cpuhp_step *sp; 1589 int target, ret; 1590 1591 ret = kstrtoint(buf, 10, &target); 1592 if (ret) 1593 return ret; 1594 1595 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL 1596 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE) 1597 return -EINVAL; 1598 #else 1599 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE) 1600 return -EINVAL; 1601 #endif 1602 1603 ret = lock_device_hotplug_sysfs(); 1604 if (ret) 1605 return ret; 1606 1607 mutex_lock(&cpuhp_state_mutex); 1608 sp = cpuhp_get_step(target); 1609 ret = !sp->name || sp->cant_stop ? -EINVAL : 0; 1610 mutex_unlock(&cpuhp_state_mutex); 1611 if (ret) 1612 goto out; 1613 1614 if (st->state < target) 1615 ret = do_cpu_up(dev->id, target); 1616 else 1617 ret = do_cpu_down(dev->id, target); 1618 out: 1619 unlock_device_hotplug(); 1620 return ret ? ret : count; 1621 } 1622 1623 static ssize_t show_cpuhp_target(struct device *dev, 1624 struct device_attribute *attr, char *buf) 1625 { 1626 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1627 1628 return sprintf(buf, "%d\n", st->target); 1629 } 1630 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target); 1631 1632 static struct attribute *cpuhp_cpu_attrs[] = { 1633 &dev_attr_state.attr, 1634 &dev_attr_target.attr, 1635 NULL 1636 }; 1637 1638 static const struct attribute_group cpuhp_cpu_attr_group = { 1639 .attrs = cpuhp_cpu_attrs, 1640 .name = "hotplug", 1641 NULL 1642 }; 1643 1644 static ssize_t show_cpuhp_states(struct device *dev, 1645 struct device_attribute *attr, char *buf) 1646 { 1647 ssize_t cur, res = 0; 1648 int i; 1649 1650 mutex_lock(&cpuhp_state_mutex); 1651 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) { 1652 struct cpuhp_step *sp = cpuhp_get_step(i); 1653 1654 if (sp->name) { 1655 cur = sprintf(buf, "%3d: %s\n", i, sp->name); 1656 buf += cur; 1657 res += cur; 1658 } 1659 } 1660 mutex_unlock(&cpuhp_state_mutex); 1661 return res; 1662 } 1663 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL); 1664 1665 static struct attribute *cpuhp_cpu_root_attrs[] = { 1666 &dev_attr_states.attr, 1667 NULL 1668 }; 1669 1670 static const struct attribute_group cpuhp_cpu_root_attr_group = { 1671 .attrs = cpuhp_cpu_root_attrs, 1672 .name = "hotplug", 1673 NULL 1674 }; 1675 1676 static int __init cpuhp_sysfs_init(void) 1677 { 1678 int cpu, ret; 1679 1680 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, 1681 &cpuhp_cpu_root_attr_group); 1682 if (ret) 1683 return ret; 1684 1685 for_each_possible_cpu(cpu) { 1686 struct device *dev = get_cpu_device(cpu); 1687 1688 if (!dev) 1689 continue; 1690 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group); 1691 if (ret) 1692 return ret; 1693 } 1694 return 0; 1695 } 1696 device_initcall(cpuhp_sysfs_init); 1697 #endif 1698 1699 /* 1700 * cpu_bit_bitmap[] is a special, "compressed" data structure that 1701 * represents all NR_CPUS bits binary values of 1<<nr. 1702 * 1703 * It is used by cpumask_of() to get a constant address to a CPU 1704 * mask value that has a single bit set only. 1705 */ 1706 1707 /* cpu_bit_bitmap[0] is empty - so we can back into it */ 1708 #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x)) 1709 #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1) 1710 #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2) 1711 #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4) 1712 1713 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = { 1714 1715 MASK_DECLARE_8(0), MASK_DECLARE_8(8), 1716 MASK_DECLARE_8(16), MASK_DECLARE_8(24), 1717 #if BITS_PER_LONG > 32 1718 MASK_DECLARE_8(32), MASK_DECLARE_8(40), 1719 MASK_DECLARE_8(48), MASK_DECLARE_8(56), 1720 #endif 1721 }; 1722 EXPORT_SYMBOL_GPL(cpu_bit_bitmap); 1723 1724 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; 1725 EXPORT_SYMBOL(cpu_all_bits); 1726 1727 #ifdef CONFIG_INIT_ALL_POSSIBLE 1728 struct cpumask __cpu_possible_mask __read_mostly 1729 = {CPU_BITS_ALL}; 1730 #else 1731 struct cpumask __cpu_possible_mask __read_mostly; 1732 #endif 1733 EXPORT_SYMBOL(__cpu_possible_mask); 1734 1735 struct cpumask __cpu_online_mask __read_mostly; 1736 EXPORT_SYMBOL(__cpu_online_mask); 1737 1738 struct cpumask __cpu_present_mask __read_mostly; 1739 EXPORT_SYMBOL(__cpu_present_mask); 1740 1741 struct cpumask __cpu_active_mask __read_mostly; 1742 EXPORT_SYMBOL(__cpu_active_mask); 1743 1744 void init_cpu_present(const struct cpumask *src) 1745 { 1746 cpumask_copy(&__cpu_present_mask, src); 1747 } 1748 1749 void init_cpu_possible(const struct cpumask *src) 1750 { 1751 cpumask_copy(&__cpu_possible_mask, src); 1752 } 1753 1754 void init_cpu_online(const struct cpumask *src) 1755 { 1756 cpumask_copy(&__cpu_online_mask, src); 1757 } 1758 1759 /* 1760 * Activate the first processor. 1761 */ 1762 void __init boot_cpu_init(void) 1763 { 1764 int cpu = smp_processor_id(); 1765 1766 /* Mark the boot cpu "present", "online" etc for SMP and UP case */ 1767 set_cpu_online(cpu, true); 1768 set_cpu_active(cpu, true); 1769 set_cpu_present(cpu, true); 1770 set_cpu_possible(cpu, true); 1771 1772 #ifdef CONFIG_SMP 1773 __boot_cpu_id = cpu; 1774 #endif 1775 } 1776 1777 /* 1778 * Must be called _AFTER_ setting up the per_cpu areas 1779 */ 1780 void __init boot_cpu_state_init(void) 1781 { 1782 per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE; 1783 } 1784