1 /* CPU control. 2 * (C) 2001, 2002, 2003, 2004 Rusty Russell 3 * 4 * This code is licenced under the GPL. 5 */ 6 #include <linux/proc_fs.h> 7 #include <linux/smp.h> 8 #include <linux/init.h> 9 #include <linux/notifier.h> 10 #include <linux/sched.h> 11 #include <linux/unistd.h> 12 #include <linux/cpu.h> 13 #include <linux/oom.h> 14 #include <linux/rcupdate.h> 15 #include <linux/export.h> 16 #include <linux/bug.h> 17 #include <linux/kthread.h> 18 #include <linux/stop_machine.h> 19 #include <linux/mutex.h> 20 #include <linux/gfp.h> 21 #include <linux/suspend.h> 22 #include <linux/lockdep.h> 23 #include <linux/tick.h> 24 #include <linux/irq.h> 25 #include <linux/smpboot.h> 26 #include <linux/relay.h> 27 #include <linux/slab.h> 28 29 #include <trace/events/power.h> 30 #define CREATE_TRACE_POINTS 31 #include <trace/events/cpuhp.h> 32 33 #include "smpboot.h" 34 35 /** 36 * cpuhp_cpu_state - Per cpu hotplug state storage 37 * @state: The current cpu state 38 * @target: The target state 39 * @thread: Pointer to the hotplug thread 40 * @should_run: Thread should execute 41 * @rollback: Perform a rollback 42 * @single: Single callback invocation 43 * @bringup: Single callback bringup or teardown selector 44 * @cb_state: The state for a single callback (install/uninstall) 45 * @result: Result of the operation 46 * @done: Signal completion to the issuer of the task 47 */ 48 struct cpuhp_cpu_state { 49 enum cpuhp_state state; 50 enum cpuhp_state target; 51 #ifdef CONFIG_SMP 52 struct task_struct *thread; 53 bool should_run; 54 bool rollback; 55 bool single; 56 bool bringup; 57 struct hlist_node *node; 58 enum cpuhp_state cb_state; 59 int result; 60 struct completion done; 61 #endif 62 }; 63 64 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state); 65 66 /** 67 * cpuhp_step - Hotplug state machine step 68 * @name: Name of the step 69 * @startup: Startup function of the step 70 * @teardown: Teardown function of the step 71 * @skip_onerr: Do not invoke the functions on error rollback 72 * Will go away once the notifiers are gone 73 * @cant_stop: Bringup/teardown can't be stopped at this step 74 */ 75 struct cpuhp_step { 76 const char *name; 77 union { 78 int (*single)(unsigned int cpu); 79 int (*multi)(unsigned int cpu, 80 struct hlist_node *node); 81 } startup; 82 union { 83 int (*single)(unsigned int cpu); 84 int (*multi)(unsigned int cpu, 85 struct hlist_node *node); 86 } teardown; 87 struct hlist_head list; 88 bool skip_onerr; 89 bool cant_stop; 90 bool multi_instance; 91 }; 92 93 static DEFINE_MUTEX(cpuhp_state_mutex); 94 static struct cpuhp_step cpuhp_bp_states[]; 95 static struct cpuhp_step cpuhp_ap_states[]; 96 97 static bool cpuhp_is_ap_state(enum cpuhp_state state) 98 { 99 /* 100 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation 101 * purposes as that state is handled explicitly in cpu_down. 102 */ 103 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU; 104 } 105 106 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state) 107 { 108 struct cpuhp_step *sp; 109 110 sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states; 111 return sp + state; 112 } 113 114 /** 115 * cpuhp_invoke_callback _ Invoke the callbacks for a given state 116 * @cpu: The cpu for which the callback should be invoked 117 * @step: The step in the state machine 118 * @bringup: True if the bringup callback should be invoked 119 * 120 * Called from cpu hotplug and from the state register machinery. 121 */ 122 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state, 123 bool bringup, struct hlist_node *node) 124 { 125 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 126 struct cpuhp_step *step = cpuhp_get_step(state); 127 int (*cbm)(unsigned int cpu, struct hlist_node *node); 128 int (*cb)(unsigned int cpu); 129 int ret, cnt; 130 131 if (!step->multi_instance) { 132 cb = bringup ? step->startup.single : step->teardown.single; 133 if (!cb) 134 return 0; 135 trace_cpuhp_enter(cpu, st->target, state, cb); 136 ret = cb(cpu); 137 trace_cpuhp_exit(cpu, st->state, state, ret); 138 return ret; 139 } 140 cbm = bringup ? step->startup.multi : step->teardown.multi; 141 if (!cbm) 142 return 0; 143 144 /* Single invocation for instance add/remove */ 145 if (node) { 146 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); 147 ret = cbm(cpu, node); 148 trace_cpuhp_exit(cpu, st->state, state, ret); 149 return ret; 150 } 151 152 /* State transition. Invoke on all instances */ 153 cnt = 0; 154 hlist_for_each(node, &step->list) { 155 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); 156 ret = cbm(cpu, node); 157 trace_cpuhp_exit(cpu, st->state, state, ret); 158 if (ret) 159 goto err; 160 cnt++; 161 } 162 return 0; 163 err: 164 /* Rollback the instances if one failed */ 165 cbm = !bringup ? step->startup.multi : step->teardown.multi; 166 if (!cbm) 167 return ret; 168 169 hlist_for_each(node, &step->list) { 170 if (!cnt--) 171 break; 172 cbm(cpu, node); 173 } 174 return ret; 175 } 176 177 #ifdef CONFIG_SMP 178 /* Serializes the updates to cpu_online_mask, cpu_present_mask */ 179 static DEFINE_MUTEX(cpu_add_remove_lock); 180 bool cpuhp_tasks_frozen; 181 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen); 182 183 /* 184 * The following two APIs (cpu_maps_update_begin/done) must be used when 185 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. 186 */ 187 void cpu_maps_update_begin(void) 188 { 189 mutex_lock(&cpu_add_remove_lock); 190 } 191 192 void cpu_maps_update_done(void) 193 { 194 mutex_unlock(&cpu_add_remove_lock); 195 } 196 197 /* If set, cpu_up and cpu_down will return -EBUSY and do nothing. 198 * Should always be manipulated under cpu_add_remove_lock 199 */ 200 static int cpu_hotplug_disabled; 201 202 #ifdef CONFIG_HOTPLUG_CPU 203 204 static struct { 205 struct task_struct *active_writer; 206 /* wait queue to wake up the active_writer */ 207 wait_queue_head_t wq; 208 /* verifies that no writer will get active while readers are active */ 209 struct mutex lock; 210 /* 211 * Also blocks the new readers during 212 * an ongoing cpu hotplug operation. 213 */ 214 atomic_t refcount; 215 216 #ifdef CONFIG_DEBUG_LOCK_ALLOC 217 struct lockdep_map dep_map; 218 #endif 219 } cpu_hotplug = { 220 .active_writer = NULL, 221 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), 222 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), 223 #ifdef CONFIG_DEBUG_LOCK_ALLOC 224 .dep_map = STATIC_LOCKDEP_MAP_INIT("cpu_hotplug.dep_map", &cpu_hotplug.dep_map), 225 #endif 226 }; 227 228 /* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */ 229 #define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map) 230 #define cpuhp_lock_acquire_tryread() \ 231 lock_map_acquire_tryread(&cpu_hotplug.dep_map) 232 #define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) 233 #define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) 234 235 236 void get_online_cpus(void) 237 { 238 might_sleep(); 239 if (cpu_hotplug.active_writer == current) 240 return; 241 cpuhp_lock_acquire_read(); 242 mutex_lock(&cpu_hotplug.lock); 243 atomic_inc(&cpu_hotplug.refcount); 244 mutex_unlock(&cpu_hotplug.lock); 245 } 246 EXPORT_SYMBOL_GPL(get_online_cpus); 247 248 void put_online_cpus(void) 249 { 250 int refcount; 251 252 if (cpu_hotplug.active_writer == current) 253 return; 254 255 refcount = atomic_dec_return(&cpu_hotplug.refcount); 256 if (WARN_ON(refcount < 0)) /* try to fix things up */ 257 atomic_inc(&cpu_hotplug.refcount); 258 259 if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq)) 260 wake_up(&cpu_hotplug.wq); 261 262 cpuhp_lock_release(); 263 264 } 265 EXPORT_SYMBOL_GPL(put_online_cpus); 266 267 /* 268 * This ensures that the hotplug operation can begin only when the 269 * refcount goes to zero. 270 * 271 * Note that during a cpu-hotplug operation, the new readers, if any, 272 * will be blocked by the cpu_hotplug.lock 273 * 274 * Since cpu_hotplug_begin() is always called after invoking 275 * cpu_maps_update_begin(), we can be sure that only one writer is active. 276 * 277 * Note that theoretically, there is a possibility of a livelock: 278 * - Refcount goes to zero, last reader wakes up the sleeping 279 * writer. 280 * - Last reader unlocks the cpu_hotplug.lock. 281 * - A new reader arrives at this moment, bumps up the refcount. 282 * - The writer acquires the cpu_hotplug.lock finds the refcount 283 * non zero and goes to sleep again. 284 * 285 * However, this is very difficult to achieve in practice since 286 * get_online_cpus() not an api which is called all that often. 287 * 288 */ 289 void cpu_hotplug_begin(void) 290 { 291 DEFINE_WAIT(wait); 292 293 cpu_hotplug.active_writer = current; 294 cpuhp_lock_acquire(); 295 296 for (;;) { 297 mutex_lock(&cpu_hotplug.lock); 298 prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE); 299 if (likely(!atomic_read(&cpu_hotplug.refcount))) 300 break; 301 mutex_unlock(&cpu_hotplug.lock); 302 schedule(); 303 } 304 finish_wait(&cpu_hotplug.wq, &wait); 305 } 306 307 void cpu_hotplug_done(void) 308 { 309 cpu_hotplug.active_writer = NULL; 310 mutex_unlock(&cpu_hotplug.lock); 311 cpuhp_lock_release(); 312 } 313 314 /* 315 * Wait for currently running CPU hotplug operations to complete (if any) and 316 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects 317 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the 318 * hotplug path before performing hotplug operations. So acquiring that lock 319 * guarantees mutual exclusion from any currently running hotplug operations. 320 */ 321 void cpu_hotplug_disable(void) 322 { 323 cpu_maps_update_begin(); 324 cpu_hotplug_disabled++; 325 cpu_maps_update_done(); 326 } 327 EXPORT_SYMBOL_GPL(cpu_hotplug_disable); 328 329 static void __cpu_hotplug_enable(void) 330 { 331 if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n")) 332 return; 333 cpu_hotplug_disabled--; 334 } 335 336 void cpu_hotplug_enable(void) 337 { 338 cpu_maps_update_begin(); 339 __cpu_hotplug_enable(); 340 cpu_maps_update_done(); 341 } 342 EXPORT_SYMBOL_GPL(cpu_hotplug_enable); 343 #endif /* CONFIG_HOTPLUG_CPU */ 344 345 /* Notifier wrappers for transitioning to state machine */ 346 347 static int bringup_wait_for_ap(unsigned int cpu) 348 { 349 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 350 351 wait_for_completion(&st->done); 352 return st->result; 353 } 354 355 static int bringup_cpu(unsigned int cpu) 356 { 357 struct task_struct *idle = idle_thread_get(cpu); 358 int ret; 359 360 /* 361 * Some architectures have to walk the irq descriptors to 362 * setup the vector space for the cpu which comes online. 363 * Prevent irq alloc/free across the bringup. 364 */ 365 irq_lock_sparse(); 366 367 /* Arch-specific enabling code. */ 368 ret = __cpu_up(cpu, idle); 369 irq_unlock_sparse(); 370 if (ret) 371 return ret; 372 ret = bringup_wait_for_ap(cpu); 373 BUG_ON(!cpu_online(cpu)); 374 return ret; 375 } 376 377 /* 378 * Hotplug state machine related functions 379 */ 380 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st) 381 { 382 for (st->state++; st->state < st->target; st->state++) { 383 struct cpuhp_step *step = cpuhp_get_step(st->state); 384 385 if (!step->skip_onerr) 386 cpuhp_invoke_callback(cpu, st->state, true, NULL); 387 } 388 } 389 390 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, 391 enum cpuhp_state target) 392 { 393 enum cpuhp_state prev_state = st->state; 394 int ret = 0; 395 396 for (; st->state > target; st->state--) { 397 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL); 398 if (ret) { 399 st->target = prev_state; 400 undo_cpu_down(cpu, st); 401 break; 402 } 403 } 404 return ret; 405 } 406 407 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st) 408 { 409 for (st->state--; st->state > st->target; st->state--) { 410 struct cpuhp_step *step = cpuhp_get_step(st->state); 411 412 if (!step->skip_onerr) 413 cpuhp_invoke_callback(cpu, st->state, false, NULL); 414 } 415 } 416 417 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, 418 enum cpuhp_state target) 419 { 420 enum cpuhp_state prev_state = st->state; 421 int ret = 0; 422 423 while (st->state < target) { 424 st->state++; 425 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL); 426 if (ret) { 427 st->target = prev_state; 428 undo_cpu_up(cpu, st); 429 break; 430 } 431 } 432 return ret; 433 } 434 435 /* 436 * The cpu hotplug threads manage the bringup and teardown of the cpus 437 */ 438 static void cpuhp_create(unsigned int cpu) 439 { 440 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 441 442 init_completion(&st->done); 443 } 444 445 static int cpuhp_should_run(unsigned int cpu) 446 { 447 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 448 449 return st->should_run; 450 } 451 452 /* Execute the teardown callbacks. Used to be CPU_DOWN_PREPARE */ 453 static int cpuhp_ap_offline(unsigned int cpu, struct cpuhp_cpu_state *st) 454 { 455 enum cpuhp_state target = max((int)st->target, CPUHP_TEARDOWN_CPU); 456 457 return cpuhp_down_callbacks(cpu, st, target); 458 } 459 460 /* Execute the online startup callbacks. Used to be CPU_ONLINE */ 461 static int cpuhp_ap_online(unsigned int cpu, struct cpuhp_cpu_state *st) 462 { 463 return cpuhp_up_callbacks(cpu, st, st->target); 464 } 465 466 /* 467 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke 468 * callbacks when a state gets [un]installed at runtime. 469 */ 470 static void cpuhp_thread_fun(unsigned int cpu) 471 { 472 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 473 int ret = 0; 474 475 /* 476 * Paired with the mb() in cpuhp_kick_ap_work and 477 * cpuhp_invoke_ap_callback, so the work set is consistent visible. 478 */ 479 smp_mb(); 480 if (!st->should_run) 481 return; 482 483 st->should_run = false; 484 485 /* Single callback invocation for [un]install ? */ 486 if (st->single) { 487 if (st->cb_state < CPUHP_AP_ONLINE) { 488 local_irq_disable(); 489 ret = cpuhp_invoke_callback(cpu, st->cb_state, 490 st->bringup, st->node); 491 local_irq_enable(); 492 } else { 493 ret = cpuhp_invoke_callback(cpu, st->cb_state, 494 st->bringup, st->node); 495 } 496 } else if (st->rollback) { 497 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE); 498 499 undo_cpu_down(cpu, st); 500 st->rollback = false; 501 } else { 502 /* Cannot happen .... */ 503 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE); 504 505 /* Regular hotplug work */ 506 if (st->state < st->target) 507 ret = cpuhp_ap_online(cpu, st); 508 else if (st->state > st->target) 509 ret = cpuhp_ap_offline(cpu, st); 510 } 511 st->result = ret; 512 complete(&st->done); 513 } 514 515 /* Invoke a single callback on a remote cpu */ 516 static int 517 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup, 518 struct hlist_node *node) 519 { 520 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 521 522 if (!cpu_online(cpu)) 523 return 0; 524 525 /* 526 * If we are up and running, use the hotplug thread. For early calls 527 * we invoke the thread function directly. 528 */ 529 if (!st->thread) 530 return cpuhp_invoke_callback(cpu, state, bringup, node); 531 532 st->cb_state = state; 533 st->single = true; 534 st->bringup = bringup; 535 st->node = node; 536 537 /* 538 * Make sure the above stores are visible before should_run becomes 539 * true. Paired with the mb() above in cpuhp_thread_fun() 540 */ 541 smp_mb(); 542 st->should_run = true; 543 wake_up_process(st->thread); 544 wait_for_completion(&st->done); 545 return st->result; 546 } 547 548 /* Regular hotplug invocation of the AP hotplug thread */ 549 static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st) 550 { 551 st->result = 0; 552 st->single = false; 553 /* 554 * Make sure the above stores are visible before should_run becomes 555 * true. Paired with the mb() above in cpuhp_thread_fun() 556 */ 557 smp_mb(); 558 st->should_run = true; 559 wake_up_process(st->thread); 560 } 561 562 static int cpuhp_kick_ap_work(unsigned int cpu) 563 { 564 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 565 enum cpuhp_state state = st->state; 566 567 trace_cpuhp_enter(cpu, st->target, state, cpuhp_kick_ap_work); 568 __cpuhp_kick_ap_work(st); 569 wait_for_completion(&st->done); 570 trace_cpuhp_exit(cpu, st->state, state, st->result); 571 return st->result; 572 } 573 574 static struct smp_hotplug_thread cpuhp_threads = { 575 .store = &cpuhp_state.thread, 576 .create = &cpuhp_create, 577 .thread_should_run = cpuhp_should_run, 578 .thread_fn = cpuhp_thread_fun, 579 .thread_comm = "cpuhp/%u", 580 .selfparking = true, 581 }; 582 583 void __init cpuhp_threads_init(void) 584 { 585 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads)); 586 kthread_unpark(this_cpu_read(cpuhp_state.thread)); 587 } 588 589 #ifdef CONFIG_HOTPLUG_CPU 590 /** 591 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU 592 * @cpu: a CPU id 593 * 594 * This function walks all processes, finds a valid mm struct for each one and 595 * then clears a corresponding bit in mm's cpumask. While this all sounds 596 * trivial, there are various non-obvious corner cases, which this function 597 * tries to solve in a safe manner. 598 * 599 * Also note that the function uses a somewhat relaxed locking scheme, so it may 600 * be called only for an already offlined CPU. 601 */ 602 void clear_tasks_mm_cpumask(int cpu) 603 { 604 struct task_struct *p; 605 606 /* 607 * This function is called after the cpu is taken down and marked 608 * offline, so its not like new tasks will ever get this cpu set in 609 * their mm mask. -- Peter Zijlstra 610 * Thus, we may use rcu_read_lock() here, instead of grabbing 611 * full-fledged tasklist_lock. 612 */ 613 WARN_ON(cpu_online(cpu)); 614 rcu_read_lock(); 615 for_each_process(p) { 616 struct task_struct *t; 617 618 /* 619 * Main thread might exit, but other threads may still have 620 * a valid mm. Find one. 621 */ 622 t = find_lock_task_mm(p); 623 if (!t) 624 continue; 625 cpumask_clear_cpu(cpu, mm_cpumask(t->mm)); 626 task_unlock(t); 627 } 628 rcu_read_unlock(); 629 } 630 631 static inline void check_for_tasks(int dead_cpu) 632 { 633 struct task_struct *g, *p; 634 635 read_lock(&tasklist_lock); 636 for_each_process_thread(g, p) { 637 if (!p->on_rq) 638 continue; 639 /* 640 * We do the check with unlocked task_rq(p)->lock. 641 * Order the reading to do not warn about a task, 642 * which was running on this cpu in the past, and 643 * it's just been woken on another cpu. 644 */ 645 rmb(); 646 if (task_cpu(p) != dead_cpu) 647 continue; 648 649 pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n", 650 p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags); 651 } 652 read_unlock(&tasklist_lock); 653 } 654 655 /* Take this CPU down. */ 656 static int take_cpu_down(void *_param) 657 { 658 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 659 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); 660 int err, cpu = smp_processor_id(); 661 662 /* Ensure this CPU doesn't handle any more interrupts. */ 663 err = __cpu_disable(); 664 if (err < 0) 665 return err; 666 667 /* 668 * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not 669 * do this step again. 670 */ 671 WARN_ON(st->state != CPUHP_TEARDOWN_CPU); 672 st->state--; 673 /* Invoke the former CPU_DYING callbacks */ 674 for (; st->state > target; st->state--) 675 cpuhp_invoke_callback(cpu, st->state, false, NULL); 676 677 /* Give up timekeeping duties */ 678 tick_handover_do_timer(); 679 /* Park the stopper thread */ 680 stop_machine_park(cpu); 681 return 0; 682 } 683 684 static int takedown_cpu(unsigned int cpu) 685 { 686 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 687 int err; 688 689 /* Park the smpboot threads */ 690 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread); 691 smpboot_park_threads(cpu); 692 693 /* 694 * Prevent irq alloc/free while the dying cpu reorganizes the 695 * interrupt affinities. 696 */ 697 irq_lock_sparse(); 698 699 /* 700 * So now all preempt/rcu users must observe !cpu_active(). 701 */ 702 err = stop_machine(take_cpu_down, NULL, cpumask_of(cpu)); 703 if (err) { 704 /* CPU refused to die */ 705 irq_unlock_sparse(); 706 /* Unpark the hotplug thread so we can rollback there */ 707 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread); 708 return err; 709 } 710 BUG_ON(cpu_online(cpu)); 711 712 /* 713 * The CPUHP_AP_SCHED_MIGRATE_DYING callback will have removed all 714 * runnable tasks from the cpu, there's only the idle task left now 715 * that the migration thread is done doing the stop_machine thing. 716 * 717 * Wait for the stop thread to go away. 718 */ 719 wait_for_completion(&st->done); 720 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); 721 722 /* Interrupts are moved away from the dying cpu, reenable alloc/free */ 723 irq_unlock_sparse(); 724 725 hotplug_cpu__broadcast_tick_pull(cpu); 726 /* This actually kills the CPU. */ 727 __cpu_die(cpu); 728 729 tick_cleanup_dead_cpu(cpu); 730 return 0; 731 } 732 733 static void cpuhp_complete_idle_dead(void *arg) 734 { 735 struct cpuhp_cpu_state *st = arg; 736 737 complete(&st->done); 738 } 739 740 void cpuhp_report_idle_dead(void) 741 { 742 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 743 744 BUG_ON(st->state != CPUHP_AP_OFFLINE); 745 rcu_report_dead(smp_processor_id()); 746 st->state = CPUHP_AP_IDLE_DEAD; 747 /* 748 * We cannot call complete after rcu_report_dead() so we delegate it 749 * to an online cpu. 750 */ 751 smp_call_function_single(cpumask_first(cpu_online_mask), 752 cpuhp_complete_idle_dead, st, 0); 753 } 754 755 #else 756 #define takedown_cpu NULL 757 #endif 758 759 #ifdef CONFIG_HOTPLUG_CPU 760 761 /* Requires cpu_add_remove_lock to be held */ 762 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, 763 enum cpuhp_state target) 764 { 765 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 766 int prev_state, ret = 0; 767 bool hasdied = false; 768 769 if (num_online_cpus() == 1) 770 return -EBUSY; 771 772 if (!cpu_present(cpu)) 773 return -EINVAL; 774 775 cpu_hotplug_begin(); 776 777 cpuhp_tasks_frozen = tasks_frozen; 778 779 prev_state = st->state; 780 st->target = target; 781 /* 782 * If the current CPU state is in the range of the AP hotplug thread, 783 * then we need to kick the thread. 784 */ 785 if (st->state > CPUHP_TEARDOWN_CPU) { 786 ret = cpuhp_kick_ap_work(cpu); 787 /* 788 * The AP side has done the error rollback already. Just 789 * return the error code.. 790 */ 791 if (ret) 792 goto out; 793 794 /* 795 * We might have stopped still in the range of the AP hotplug 796 * thread. Nothing to do anymore. 797 */ 798 if (st->state > CPUHP_TEARDOWN_CPU) 799 goto out; 800 } 801 /* 802 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need 803 * to do the further cleanups. 804 */ 805 ret = cpuhp_down_callbacks(cpu, st, target); 806 if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) { 807 st->target = prev_state; 808 st->rollback = true; 809 cpuhp_kick_ap_work(cpu); 810 } 811 812 hasdied = prev_state != st->state && st->state == CPUHP_OFFLINE; 813 out: 814 cpu_hotplug_done(); 815 return ret; 816 } 817 818 static int do_cpu_down(unsigned int cpu, enum cpuhp_state target) 819 { 820 int err; 821 822 cpu_maps_update_begin(); 823 824 if (cpu_hotplug_disabled) { 825 err = -EBUSY; 826 goto out; 827 } 828 829 err = _cpu_down(cpu, 0, target); 830 831 out: 832 cpu_maps_update_done(); 833 return err; 834 } 835 int cpu_down(unsigned int cpu) 836 { 837 return do_cpu_down(cpu, CPUHP_OFFLINE); 838 } 839 EXPORT_SYMBOL(cpu_down); 840 #endif /*CONFIG_HOTPLUG_CPU*/ 841 842 /** 843 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU 844 * @cpu: cpu that just started 845 * 846 * It must be called by the arch code on the new cpu, before the new cpu 847 * enables interrupts and before the "boot" cpu returns from __cpu_up(). 848 */ 849 void notify_cpu_starting(unsigned int cpu) 850 { 851 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 852 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); 853 854 rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */ 855 while (st->state < target) { 856 st->state++; 857 cpuhp_invoke_callback(cpu, st->state, true, NULL); 858 } 859 } 860 861 /* 862 * Called from the idle task. We need to set active here, so we can kick off 863 * the stopper thread and unpark the smpboot threads. If the target state is 864 * beyond CPUHP_AP_ONLINE_IDLE we kick cpuhp thread and let it bring up the 865 * cpu further. 866 */ 867 void cpuhp_online_idle(enum cpuhp_state state) 868 { 869 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 870 unsigned int cpu = smp_processor_id(); 871 872 /* Happens for the boot cpu */ 873 if (state != CPUHP_AP_ONLINE_IDLE) 874 return; 875 876 st->state = CPUHP_AP_ONLINE_IDLE; 877 878 /* Unpark the stopper thread and the hotplug thread of this cpu */ 879 stop_machine_unpark(cpu); 880 kthread_unpark(st->thread); 881 882 /* Should we go further up ? */ 883 if (st->target > CPUHP_AP_ONLINE_IDLE) 884 __cpuhp_kick_ap_work(st); 885 else 886 complete(&st->done); 887 } 888 889 /* Requires cpu_add_remove_lock to be held */ 890 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target) 891 { 892 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 893 struct task_struct *idle; 894 int ret = 0; 895 896 cpu_hotplug_begin(); 897 898 if (!cpu_present(cpu)) { 899 ret = -EINVAL; 900 goto out; 901 } 902 903 /* 904 * The caller of do_cpu_up might have raced with another 905 * caller. Ignore it for now. 906 */ 907 if (st->state >= target) 908 goto out; 909 910 if (st->state == CPUHP_OFFLINE) { 911 /* Let it fail before we try to bring the cpu up */ 912 idle = idle_thread_get(cpu); 913 if (IS_ERR(idle)) { 914 ret = PTR_ERR(idle); 915 goto out; 916 } 917 } 918 919 cpuhp_tasks_frozen = tasks_frozen; 920 921 st->target = target; 922 /* 923 * If the current CPU state is in the range of the AP hotplug thread, 924 * then we need to kick the thread once more. 925 */ 926 if (st->state > CPUHP_BRINGUP_CPU) { 927 ret = cpuhp_kick_ap_work(cpu); 928 /* 929 * The AP side has done the error rollback already. Just 930 * return the error code.. 931 */ 932 if (ret) 933 goto out; 934 } 935 936 /* 937 * Try to reach the target state. We max out on the BP at 938 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is 939 * responsible for bringing it up to the target state. 940 */ 941 target = min((int)target, CPUHP_BRINGUP_CPU); 942 ret = cpuhp_up_callbacks(cpu, st, target); 943 out: 944 cpu_hotplug_done(); 945 return ret; 946 } 947 948 static int do_cpu_up(unsigned int cpu, enum cpuhp_state target) 949 { 950 int err = 0; 951 952 if (!cpu_possible(cpu)) { 953 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n", 954 cpu); 955 #if defined(CONFIG_IA64) 956 pr_err("please check additional_cpus= boot parameter\n"); 957 #endif 958 return -EINVAL; 959 } 960 961 err = try_online_node(cpu_to_node(cpu)); 962 if (err) 963 return err; 964 965 cpu_maps_update_begin(); 966 967 if (cpu_hotplug_disabled) { 968 err = -EBUSY; 969 goto out; 970 } 971 972 err = _cpu_up(cpu, 0, target); 973 out: 974 cpu_maps_update_done(); 975 return err; 976 } 977 978 int cpu_up(unsigned int cpu) 979 { 980 return do_cpu_up(cpu, CPUHP_ONLINE); 981 } 982 EXPORT_SYMBOL_GPL(cpu_up); 983 984 #ifdef CONFIG_PM_SLEEP_SMP 985 static cpumask_var_t frozen_cpus; 986 987 int freeze_secondary_cpus(int primary) 988 { 989 int cpu, error = 0; 990 991 cpu_maps_update_begin(); 992 if (!cpu_online(primary)) 993 primary = cpumask_first(cpu_online_mask); 994 /* 995 * We take down all of the non-boot CPUs in one shot to avoid races 996 * with the userspace trying to use the CPU hotplug at the same time 997 */ 998 cpumask_clear(frozen_cpus); 999 1000 pr_info("Disabling non-boot CPUs ...\n"); 1001 for_each_online_cpu(cpu) { 1002 if (cpu == primary) 1003 continue; 1004 trace_suspend_resume(TPS("CPU_OFF"), cpu, true); 1005 error = _cpu_down(cpu, 1, CPUHP_OFFLINE); 1006 trace_suspend_resume(TPS("CPU_OFF"), cpu, false); 1007 if (!error) 1008 cpumask_set_cpu(cpu, frozen_cpus); 1009 else { 1010 pr_err("Error taking CPU%d down: %d\n", cpu, error); 1011 break; 1012 } 1013 } 1014 1015 if (!error) 1016 BUG_ON(num_online_cpus() > 1); 1017 else 1018 pr_err("Non-boot CPUs are not disabled\n"); 1019 1020 /* 1021 * Make sure the CPUs won't be enabled by someone else. We need to do 1022 * this even in case of failure as all disable_nonboot_cpus() users are 1023 * supposed to do enable_nonboot_cpus() on the failure path. 1024 */ 1025 cpu_hotplug_disabled++; 1026 1027 cpu_maps_update_done(); 1028 return error; 1029 } 1030 1031 void __weak arch_enable_nonboot_cpus_begin(void) 1032 { 1033 } 1034 1035 void __weak arch_enable_nonboot_cpus_end(void) 1036 { 1037 } 1038 1039 void enable_nonboot_cpus(void) 1040 { 1041 int cpu, error; 1042 1043 /* Allow everyone to use the CPU hotplug again */ 1044 cpu_maps_update_begin(); 1045 __cpu_hotplug_enable(); 1046 if (cpumask_empty(frozen_cpus)) 1047 goto out; 1048 1049 pr_info("Enabling non-boot CPUs ...\n"); 1050 1051 arch_enable_nonboot_cpus_begin(); 1052 1053 for_each_cpu(cpu, frozen_cpus) { 1054 trace_suspend_resume(TPS("CPU_ON"), cpu, true); 1055 error = _cpu_up(cpu, 1, CPUHP_ONLINE); 1056 trace_suspend_resume(TPS("CPU_ON"), cpu, false); 1057 if (!error) { 1058 pr_info("CPU%d is up\n", cpu); 1059 continue; 1060 } 1061 pr_warn("Error taking CPU%d up: %d\n", cpu, error); 1062 } 1063 1064 arch_enable_nonboot_cpus_end(); 1065 1066 cpumask_clear(frozen_cpus); 1067 out: 1068 cpu_maps_update_done(); 1069 } 1070 1071 static int __init alloc_frozen_cpus(void) 1072 { 1073 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO)) 1074 return -ENOMEM; 1075 return 0; 1076 } 1077 core_initcall(alloc_frozen_cpus); 1078 1079 /* 1080 * When callbacks for CPU hotplug notifications are being executed, we must 1081 * ensure that the state of the system with respect to the tasks being frozen 1082 * or not, as reported by the notification, remains unchanged *throughout the 1083 * duration* of the execution of the callbacks. 1084 * Hence we need to prevent the freezer from racing with regular CPU hotplug. 1085 * 1086 * This synchronization is implemented by mutually excluding regular CPU 1087 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/ 1088 * Hibernate notifications. 1089 */ 1090 static int 1091 cpu_hotplug_pm_callback(struct notifier_block *nb, 1092 unsigned long action, void *ptr) 1093 { 1094 switch (action) { 1095 1096 case PM_SUSPEND_PREPARE: 1097 case PM_HIBERNATION_PREPARE: 1098 cpu_hotplug_disable(); 1099 break; 1100 1101 case PM_POST_SUSPEND: 1102 case PM_POST_HIBERNATION: 1103 cpu_hotplug_enable(); 1104 break; 1105 1106 default: 1107 return NOTIFY_DONE; 1108 } 1109 1110 return NOTIFY_OK; 1111 } 1112 1113 1114 static int __init cpu_hotplug_pm_sync_init(void) 1115 { 1116 /* 1117 * cpu_hotplug_pm_callback has higher priority than x86 1118 * bsp_pm_callback which depends on cpu_hotplug_pm_callback 1119 * to disable cpu hotplug to avoid cpu hotplug race. 1120 */ 1121 pm_notifier(cpu_hotplug_pm_callback, 0); 1122 return 0; 1123 } 1124 core_initcall(cpu_hotplug_pm_sync_init); 1125 1126 #endif /* CONFIG_PM_SLEEP_SMP */ 1127 1128 #endif /* CONFIG_SMP */ 1129 1130 /* Boot processor state steps */ 1131 static struct cpuhp_step cpuhp_bp_states[] = { 1132 [CPUHP_OFFLINE] = { 1133 .name = "offline", 1134 .startup.single = NULL, 1135 .teardown.single = NULL, 1136 }, 1137 #ifdef CONFIG_SMP 1138 [CPUHP_CREATE_THREADS]= { 1139 .name = "threads:prepare", 1140 .startup.single = smpboot_create_threads, 1141 .teardown.single = NULL, 1142 .cant_stop = true, 1143 }, 1144 [CPUHP_PERF_PREPARE] = { 1145 .name = "perf:prepare", 1146 .startup.single = perf_event_init_cpu, 1147 .teardown.single = perf_event_exit_cpu, 1148 }, 1149 [CPUHP_WORKQUEUE_PREP] = { 1150 .name = "workqueue:prepare", 1151 .startup.single = workqueue_prepare_cpu, 1152 .teardown.single = NULL, 1153 }, 1154 [CPUHP_HRTIMERS_PREPARE] = { 1155 .name = "hrtimers:prepare", 1156 .startup.single = hrtimers_prepare_cpu, 1157 .teardown.single = hrtimers_dead_cpu, 1158 }, 1159 [CPUHP_SMPCFD_PREPARE] = { 1160 .name = "smpcfd:prepare", 1161 .startup.single = smpcfd_prepare_cpu, 1162 .teardown.single = smpcfd_dead_cpu, 1163 }, 1164 [CPUHP_RELAY_PREPARE] = { 1165 .name = "relay:prepare", 1166 .startup.single = relay_prepare_cpu, 1167 .teardown.single = NULL, 1168 }, 1169 [CPUHP_SLAB_PREPARE] = { 1170 .name = "slab:prepare", 1171 .startup.single = slab_prepare_cpu, 1172 .teardown.single = slab_dead_cpu, 1173 }, 1174 [CPUHP_RCUTREE_PREP] = { 1175 .name = "RCU/tree:prepare", 1176 .startup.single = rcutree_prepare_cpu, 1177 .teardown.single = rcutree_dead_cpu, 1178 }, 1179 /* 1180 * On the tear-down path, timers_dead_cpu() must be invoked 1181 * before blk_mq_queue_reinit_notify() from notify_dead(), 1182 * otherwise a RCU stall occurs. 1183 */ 1184 [CPUHP_TIMERS_DEAD] = { 1185 .name = "timers:dead", 1186 .startup.single = NULL, 1187 .teardown.single = timers_dead_cpu, 1188 }, 1189 /* Kicks the plugged cpu into life */ 1190 [CPUHP_BRINGUP_CPU] = { 1191 .name = "cpu:bringup", 1192 .startup.single = bringup_cpu, 1193 .teardown.single = NULL, 1194 .cant_stop = true, 1195 }, 1196 [CPUHP_AP_SMPCFD_DYING] = { 1197 .name = "smpcfd:dying", 1198 .startup.single = NULL, 1199 .teardown.single = smpcfd_dying_cpu, 1200 }, 1201 /* 1202 * Handled on controll processor until the plugged processor manages 1203 * this itself. 1204 */ 1205 [CPUHP_TEARDOWN_CPU] = { 1206 .name = "cpu:teardown", 1207 .startup.single = NULL, 1208 .teardown.single = takedown_cpu, 1209 .cant_stop = true, 1210 }, 1211 #else 1212 [CPUHP_BRINGUP_CPU] = { }, 1213 #endif 1214 }; 1215 1216 /* Application processor state steps */ 1217 static struct cpuhp_step cpuhp_ap_states[] = { 1218 #ifdef CONFIG_SMP 1219 /* Final state before CPU kills itself */ 1220 [CPUHP_AP_IDLE_DEAD] = { 1221 .name = "idle:dead", 1222 }, 1223 /* 1224 * Last state before CPU enters the idle loop to die. Transient state 1225 * for synchronization. 1226 */ 1227 [CPUHP_AP_OFFLINE] = { 1228 .name = "ap:offline", 1229 .cant_stop = true, 1230 }, 1231 /* First state is scheduler control. Interrupts are disabled */ 1232 [CPUHP_AP_SCHED_STARTING] = { 1233 .name = "sched:starting", 1234 .startup.single = sched_cpu_starting, 1235 .teardown.single = sched_cpu_dying, 1236 }, 1237 [CPUHP_AP_RCUTREE_DYING] = { 1238 .name = "RCU/tree:dying", 1239 .startup.single = NULL, 1240 .teardown.single = rcutree_dying_cpu, 1241 }, 1242 /* Entry state on starting. Interrupts enabled from here on. Transient 1243 * state for synchronsization */ 1244 [CPUHP_AP_ONLINE] = { 1245 .name = "ap:online", 1246 }, 1247 /* Handle smpboot threads park/unpark */ 1248 [CPUHP_AP_SMPBOOT_THREADS] = { 1249 .name = "smpboot/threads:online", 1250 .startup.single = smpboot_unpark_threads, 1251 .teardown.single = NULL, 1252 }, 1253 [CPUHP_AP_PERF_ONLINE] = { 1254 .name = "perf:online", 1255 .startup.single = perf_event_init_cpu, 1256 .teardown.single = perf_event_exit_cpu, 1257 }, 1258 [CPUHP_AP_WORKQUEUE_ONLINE] = { 1259 .name = "workqueue:online", 1260 .startup.single = workqueue_online_cpu, 1261 .teardown.single = workqueue_offline_cpu, 1262 }, 1263 [CPUHP_AP_RCUTREE_ONLINE] = { 1264 .name = "RCU/tree:online", 1265 .startup.single = rcutree_online_cpu, 1266 .teardown.single = rcutree_offline_cpu, 1267 }, 1268 #endif 1269 /* 1270 * The dynamically registered state space is here 1271 */ 1272 1273 #ifdef CONFIG_SMP 1274 /* Last state is scheduler control setting the cpu active */ 1275 [CPUHP_AP_ACTIVE] = { 1276 .name = "sched:active", 1277 .startup.single = sched_cpu_activate, 1278 .teardown.single = sched_cpu_deactivate, 1279 }, 1280 #endif 1281 1282 /* CPU is fully up and running. */ 1283 [CPUHP_ONLINE] = { 1284 .name = "online", 1285 .startup.single = NULL, 1286 .teardown.single = NULL, 1287 }, 1288 }; 1289 1290 /* Sanity check for callbacks */ 1291 static int cpuhp_cb_check(enum cpuhp_state state) 1292 { 1293 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE) 1294 return -EINVAL; 1295 return 0; 1296 } 1297 1298 /* 1299 * Returns a free for dynamic slot assignment of the Online state. The states 1300 * are protected by the cpuhp_slot_states mutex and an empty slot is identified 1301 * by having no name assigned. 1302 */ 1303 static int cpuhp_reserve_state(enum cpuhp_state state) 1304 { 1305 enum cpuhp_state i; 1306 1307 for (i = CPUHP_AP_ONLINE_DYN; i <= CPUHP_AP_ONLINE_DYN_END; i++) { 1308 if (!cpuhp_ap_states[i].name) 1309 return i; 1310 } 1311 WARN(1, "No more dynamic states available for CPU hotplug\n"); 1312 return -ENOSPC; 1313 } 1314 1315 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name, 1316 int (*startup)(unsigned int cpu), 1317 int (*teardown)(unsigned int cpu), 1318 bool multi_instance) 1319 { 1320 /* (Un)Install the callbacks for further cpu hotplug operations */ 1321 struct cpuhp_step *sp; 1322 int ret = 0; 1323 1324 mutex_lock(&cpuhp_state_mutex); 1325 1326 if (state == CPUHP_AP_ONLINE_DYN) { 1327 ret = cpuhp_reserve_state(state); 1328 if (ret < 0) 1329 goto out; 1330 state = ret; 1331 } 1332 sp = cpuhp_get_step(state); 1333 if (name && sp->name) { 1334 ret = -EBUSY; 1335 goto out; 1336 } 1337 sp->startup.single = startup; 1338 sp->teardown.single = teardown; 1339 sp->name = name; 1340 sp->multi_instance = multi_instance; 1341 INIT_HLIST_HEAD(&sp->list); 1342 out: 1343 mutex_unlock(&cpuhp_state_mutex); 1344 return ret; 1345 } 1346 1347 static void *cpuhp_get_teardown_cb(enum cpuhp_state state) 1348 { 1349 return cpuhp_get_step(state)->teardown.single; 1350 } 1351 1352 /* 1353 * Call the startup/teardown function for a step either on the AP or 1354 * on the current CPU. 1355 */ 1356 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup, 1357 struct hlist_node *node) 1358 { 1359 struct cpuhp_step *sp = cpuhp_get_step(state); 1360 int ret; 1361 1362 if ((bringup && !sp->startup.single) || 1363 (!bringup && !sp->teardown.single)) 1364 return 0; 1365 /* 1366 * The non AP bound callbacks can fail on bringup. On teardown 1367 * e.g. module removal we crash for now. 1368 */ 1369 #ifdef CONFIG_SMP 1370 if (cpuhp_is_ap_state(state)) 1371 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node); 1372 else 1373 ret = cpuhp_invoke_callback(cpu, state, bringup, node); 1374 #else 1375 ret = cpuhp_invoke_callback(cpu, state, bringup, node); 1376 #endif 1377 BUG_ON(ret && !bringup); 1378 return ret; 1379 } 1380 1381 /* 1382 * Called from __cpuhp_setup_state on a recoverable failure. 1383 * 1384 * Note: The teardown callbacks for rollback are not allowed to fail! 1385 */ 1386 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state, 1387 struct hlist_node *node) 1388 { 1389 int cpu; 1390 1391 /* Roll back the already executed steps on the other cpus */ 1392 for_each_present_cpu(cpu) { 1393 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1394 int cpustate = st->state; 1395 1396 if (cpu >= failedcpu) 1397 break; 1398 1399 /* Did we invoke the startup call on that cpu ? */ 1400 if (cpustate >= state) 1401 cpuhp_issue_call(cpu, state, false, node); 1402 } 1403 } 1404 1405 int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node, 1406 bool invoke) 1407 { 1408 struct cpuhp_step *sp; 1409 int cpu; 1410 int ret; 1411 1412 sp = cpuhp_get_step(state); 1413 if (sp->multi_instance == false) 1414 return -EINVAL; 1415 1416 get_online_cpus(); 1417 1418 if (!invoke || !sp->startup.multi) 1419 goto add_node; 1420 1421 /* 1422 * Try to call the startup callback for each present cpu 1423 * depending on the hotplug state of the cpu. 1424 */ 1425 for_each_present_cpu(cpu) { 1426 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1427 int cpustate = st->state; 1428 1429 if (cpustate < state) 1430 continue; 1431 1432 ret = cpuhp_issue_call(cpu, state, true, node); 1433 if (ret) { 1434 if (sp->teardown.multi) 1435 cpuhp_rollback_install(cpu, state, node); 1436 goto err; 1437 } 1438 } 1439 add_node: 1440 ret = 0; 1441 mutex_lock(&cpuhp_state_mutex); 1442 hlist_add_head(node, &sp->list); 1443 mutex_unlock(&cpuhp_state_mutex); 1444 1445 err: 1446 put_online_cpus(); 1447 return ret; 1448 } 1449 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance); 1450 1451 /** 1452 * __cpuhp_setup_state - Setup the callbacks for an hotplug machine state 1453 * @state: The state to setup 1454 * @invoke: If true, the startup function is invoked for cpus where 1455 * cpu state >= @state 1456 * @startup: startup callback function 1457 * @teardown: teardown callback function 1458 * @multi_instance: State is set up for multiple instances which get 1459 * added afterwards. 1460 * 1461 * Returns: 1462 * On success: 1463 * Positive state number if @state is CPUHP_AP_ONLINE_DYN 1464 * 0 for all other states 1465 * On failure: proper (negative) error code 1466 */ 1467 int __cpuhp_setup_state(enum cpuhp_state state, 1468 const char *name, bool invoke, 1469 int (*startup)(unsigned int cpu), 1470 int (*teardown)(unsigned int cpu), 1471 bool multi_instance) 1472 { 1473 int cpu, ret = 0; 1474 1475 if (cpuhp_cb_check(state) || !name) 1476 return -EINVAL; 1477 1478 get_online_cpus(); 1479 1480 ret = cpuhp_store_callbacks(state, name, startup, teardown, 1481 multi_instance); 1482 1483 if (ret || !invoke || !startup) 1484 goto out; 1485 1486 /* 1487 * Try to call the startup callback for each present cpu 1488 * depending on the hotplug state of the cpu. 1489 */ 1490 for_each_present_cpu(cpu) { 1491 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1492 int cpustate = st->state; 1493 1494 if (cpustate < state) 1495 continue; 1496 1497 ret = cpuhp_issue_call(cpu, state, true, NULL); 1498 if (ret) { 1499 if (teardown) 1500 cpuhp_rollback_install(cpu, state, NULL); 1501 cpuhp_store_callbacks(state, NULL, NULL, NULL, false); 1502 goto out; 1503 } 1504 } 1505 out: 1506 put_online_cpus(); 1507 /* 1508 * If the requested state is CPUHP_AP_ONLINE_DYN, return the 1509 * dynamically allocated state in case of success. 1510 */ 1511 if (!ret && state == CPUHP_AP_ONLINE_DYN) 1512 return state; 1513 return ret; 1514 } 1515 EXPORT_SYMBOL(__cpuhp_setup_state); 1516 1517 int __cpuhp_state_remove_instance(enum cpuhp_state state, 1518 struct hlist_node *node, bool invoke) 1519 { 1520 struct cpuhp_step *sp = cpuhp_get_step(state); 1521 int cpu; 1522 1523 BUG_ON(cpuhp_cb_check(state)); 1524 1525 if (!sp->multi_instance) 1526 return -EINVAL; 1527 1528 get_online_cpus(); 1529 if (!invoke || !cpuhp_get_teardown_cb(state)) 1530 goto remove; 1531 /* 1532 * Call the teardown callback for each present cpu depending 1533 * on the hotplug state of the cpu. This function is not 1534 * allowed to fail currently! 1535 */ 1536 for_each_present_cpu(cpu) { 1537 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1538 int cpustate = st->state; 1539 1540 if (cpustate >= state) 1541 cpuhp_issue_call(cpu, state, false, node); 1542 } 1543 1544 remove: 1545 mutex_lock(&cpuhp_state_mutex); 1546 hlist_del(node); 1547 mutex_unlock(&cpuhp_state_mutex); 1548 put_online_cpus(); 1549 1550 return 0; 1551 } 1552 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance); 1553 /** 1554 * __cpuhp_remove_state - Remove the callbacks for an hotplug machine state 1555 * @state: The state to remove 1556 * @invoke: If true, the teardown function is invoked for cpus where 1557 * cpu state >= @state 1558 * 1559 * The teardown callback is currently not allowed to fail. Think 1560 * about module removal! 1561 */ 1562 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke) 1563 { 1564 struct cpuhp_step *sp = cpuhp_get_step(state); 1565 int cpu; 1566 1567 BUG_ON(cpuhp_cb_check(state)); 1568 1569 get_online_cpus(); 1570 1571 if (sp->multi_instance) { 1572 WARN(!hlist_empty(&sp->list), 1573 "Error: Removing state %d which has instances left.\n", 1574 state); 1575 goto remove; 1576 } 1577 1578 if (!invoke || !cpuhp_get_teardown_cb(state)) 1579 goto remove; 1580 1581 /* 1582 * Call the teardown callback for each present cpu depending 1583 * on the hotplug state of the cpu. This function is not 1584 * allowed to fail currently! 1585 */ 1586 for_each_present_cpu(cpu) { 1587 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1588 int cpustate = st->state; 1589 1590 if (cpustate >= state) 1591 cpuhp_issue_call(cpu, state, false, NULL); 1592 } 1593 remove: 1594 cpuhp_store_callbacks(state, NULL, NULL, NULL, false); 1595 put_online_cpus(); 1596 } 1597 EXPORT_SYMBOL(__cpuhp_remove_state); 1598 1599 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU) 1600 static ssize_t show_cpuhp_state(struct device *dev, 1601 struct device_attribute *attr, char *buf) 1602 { 1603 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1604 1605 return sprintf(buf, "%d\n", st->state); 1606 } 1607 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL); 1608 1609 static ssize_t write_cpuhp_target(struct device *dev, 1610 struct device_attribute *attr, 1611 const char *buf, size_t count) 1612 { 1613 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1614 struct cpuhp_step *sp; 1615 int target, ret; 1616 1617 ret = kstrtoint(buf, 10, &target); 1618 if (ret) 1619 return ret; 1620 1621 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL 1622 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE) 1623 return -EINVAL; 1624 #else 1625 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE) 1626 return -EINVAL; 1627 #endif 1628 1629 ret = lock_device_hotplug_sysfs(); 1630 if (ret) 1631 return ret; 1632 1633 mutex_lock(&cpuhp_state_mutex); 1634 sp = cpuhp_get_step(target); 1635 ret = !sp->name || sp->cant_stop ? -EINVAL : 0; 1636 mutex_unlock(&cpuhp_state_mutex); 1637 if (ret) 1638 return ret; 1639 1640 if (st->state < target) 1641 ret = do_cpu_up(dev->id, target); 1642 else 1643 ret = do_cpu_down(dev->id, target); 1644 1645 unlock_device_hotplug(); 1646 return ret ? ret : count; 1647 } 1648 1649 static ssize_t show_cpuhp_target(struct device *dev, 1650 struct device_attribute *attr, char *buf) 1651 { 1652 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1653 1654 return sprintf(buf, "%d\n", st->target); 1655 } 1656 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target); 1657 1658 static struct attribute *cpuhp_cpu_attrs[] = { 1659 &dev_attr_state.attr, 1660 &dev_attr_target.attr, 1661 NULL 1662 }; 1663 1664 static struct attribute_group cpuhp_cpu_attr_group = { 1665 .attrs = cpuhp_cpu_attrs, 1666 .name = "hotplug", 1667 NULL 1668 }; 1669 1670 static ssize_t show_cpuhp_states(struct device *dev, 1671 struct device_attribute *attr, char *buf) 1672 { 1673 ssize_t cur, res = 0; 1674 int i; 1675 1676 mutex_lock(&cpuhp_state_mutex); 1677 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) { 1678 struct cpuhp_step *sp = cpuhp_get_step(i); 1679 1680 if (sp->name) { 1681 cur = sprintf(buf, "%3d: %s\n", i, sp->name); 1682 buf += cur; 1683 res += cur; 1684 } 1685 } 1686 mutex_unlock(&cpuhp_state_mutex); 1687 return res; 1688 } 1689 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL); 1690 1691 static struct attribute *cpuhp_cpu_root_attrs[] = { 1692 &dev_attr_states.attr, 1693 NULL 1694 }; 1695 1696 static struct attribute_group cpuhp_cpu_root_attr_group = { 1697 .attrs = cpuhp_cpu_root_attrs, 1698 .name = "hotplug", 1699 NULL 1700 }; 1701 1702 static int __init cpuhp_sysfs_init(void) 1703 { 1704 int cpu, ret; 1705 1706 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, 1707 &cpuhp_cpu_root_attr_group); 1708 if (ret) 1709 return ret; 1710 1711 for_each_possible_cpu(cpu) { 1712 struct device *dev = get_cpu_device(cpu); 1713 1714 if (!dev) 1715 continue; 1716 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group); 1717 if (ret) 1718 return ret; 1719 } 1720 return 0; 1721 } 1722 device_initcall(cpuhp_sysfs_init); 1723 #endif 1724 1725 /* 1726 * cpu_bit_bitmap[] is a special, "compressed" data structure that 1727 * represents all NR_CPUS bits binary values of 1<<nr. 1728 * 1729 * It is used by cpumask_of() to get a constant address to a CPU 1730 * mask value that has a single bit set only. 1731 */ 1732 1733 /* cpu_bit_bitmap[0] is empty - so we can back into it */ 1734 #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x)) 1735 #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1) 1736 #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2) 1737 #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4) 1738 1739 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = { 1740 1741 MASK_DECLARE_8(0), MASK_DECLARE_8(8), 1742 MASK_DECLARE_8(16), MASK_DECLARE_8(24), 1743 #if BITS_PER_LONG > 32 1744 MASK_DECLARE_8(32), MASK_DECLARE_8(40), 1745 MASK_DECLARE_8(48), MASK_DECLARE_8(56), 1746 #endif 1747 }; 1748 EXPORT_SYMBOL_GPL(cpu_bit_bitmap); 1749 1750 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; 1751 EXPORT_SYMBOL(cpu_all_bits); 1752 1753 #ifdef CONFIG_INIT_ALL_POSSIBLE 1754 struct cpumask __cpu_possible_mask __read_mostly 1755 = {CPU_BITS_ALL}; 1756 #else 1757 struct cpumask __cpu_possible_mask __read_mostly; 1758 #endif 1759 EXPORT_SYMBOL(__cpu_possible_mask); 1760 1761 struct cpumask __cpu_online_mask __read_mostly; 1762 EXPORT_SYMBOL(__cpu_online_mask); 1763 1764 struct cpumask __cpu_present_mask __read_mostly; 1765 EXPORT_SYMBOL(__cpu_present_mask); 1766 1767 struct cpumask __cpu_active_mask __read_mostly; 1768 EXPORT_SYMBOL(__cpu_active_mask); 1769 1770 void init_cpu_present(const struct cpumask *src) 1771 { 1772 cpumask_copy(&__cpu_present_mask, src); 1773 } 1774 1775 void init_cpu_possible(const struct cpumask *src) 1776 { 1777 cpumask_copy(&__cpu_possible_mask, src); 1778 } 1779 1780 void init_cpu_online(const struct cpumask *src) 1781 { 1782 cpumask_copy(&__cpu_online_mask, src); 1783 } 1784 1785 /* 1786 * Activate the first processor. 1787 */ 1788 void __init boot_cpu_init(void) 1789 { 1790 int cpu = smp_processor_id(); 1791 1792 /* Mark the boot cpu "present", "online" etc for SMP and UP case */ 1793 set_cpu_online(cpu, true); 1794 set_cpu_active(cpu, true); 1795 set_cpu_present(cpu, true); 1796 set_cpu_possible(cpu, true); 1797 } 1798 1799 /* 1800 * Must be called _AFTER_ setting up the per_cpu areas 1801 */ 1802 void __init boot_cpu_state_init(void) 1803 { 1804 per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE; 1805 } 1806