1 /* CPU control. 2 * (C) 2001, 2002, 2003, 2004 Rusty Russell 3 * 4 * This code is licenced under the GPL. 5 */ 6 #include <linux/proc_fs.h> 7 #include <linux/smp.h> 8 #include <linux/init.h> 9 #include <linux/notifier.h> 10 #include <linux/sched.h> 11 #include <linux/unistd.h> 12 #include <linux/cpu.h> 13 #include <linux/oom.h> 14 #include <linux/rcupdate.h> 15 #include <linux/export.h> 16 #include <linux/bug.h> 17 #include <linux/kthread.h> 18 #include <linux/stop_machine.h> 19 #include <linux/mutex.h> 20 #include <linux/gfp.h> 21 #include <linux/suspend.h> 22 #include <linux/lockdep.h> 23 #include <linux/tick.h> 24 #include <linux/irq.h> 25 #include <linux/smpboot.h> 26 27 #include <trace/events/power.h> 28 #define CREATE_TRACE_POINTS 29 #include <trace/events/cpuhp.h> 30 31 #include "smpboot.h" 32 33 /** 34 * cpuhp_cpu_state - Per cpu hotplug state storage 35 * @state: The current cpu state 36 * @target: The target state 37 * @thread: Pointer to the hotplug thread 38 * @should_run: Thread should execute 39 * @rollback: Perform a rollback 40 * @cb_stat: The state for a single callback (install/uninstall) 41 * @cb: Single callback function (install/uninstall) 42 * @result: Result of the operation 43 * @done: Signal completion to the issuer of the task 44 */ 45 struct cpuhp_cpu_state { 46 enum cpuhp_state state; 47 enum cpuhp_state target; 48 #ifdef CONFIG_SMP 49 struct task_struct *thread; 50 bool should_run; 51 bool rollback; 52 enum cpuhp_state cb_state; 53 int (*cb)(unsigned int cpu); 54 int result; 55 struct completion done; 56 #endif 57 }; 58 59 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state); 60 61 /** 62 * cpuhp_step - Hotplug state machine step 63 * @name: Name of the step 64 * @startup: Startup function of the step 65 * @teardown: Teardown function of the step 66 * @skip_onerr: Do not invoke the functions on error rollback 67 * Will go away once the notifiers are gone 68 * @cant_stop: Bringup/teardown can't be stopped at this step 69 */ 70 struct cpuhp_step { 71 const char *name; 72 int (*startup)(unsigned int cpu); 73 int (*teardown)(unsigned int cpu); 74 bool skip_onerr; 75 bool cant_stop; 76 }; 77 78 static DEFINE_MUTEX(cpuhp_state_mutex); 79 static struct cpuhp_step cpuhp_bp_states[]; 80 static struct cpuhp_step cpuhp_ap_states[]; 81 82 /** 83 * cpuhp_invoke_callback _ Invoke the callbacks for a given state 84 * @cpu: The cpu for which the callback should be invoked 85 * @step: The step in the state machine 86 * @cb: The callback function to invoke 87 * 88 * Called from cpu hotplug and from the state register machinery 89 */ 90 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state step, 91 int (*cb)(unsigned int)) 92 { 93 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 94 int ret = 0; 95 96 if (cb) { 97 trace_cpuhp_enter(cpu, st->target, step, cb); 98 ret = cb(cpu); 99 trace_cpuhp_exit(cpu, st->state, step, ret); 100 } 101 return ret; 102 } 103 104 #ifdef CONFIG_SMP 105 /* Serializes the updates to cpu_online_mask, cpu_present_mask */ 106 static DEFINE_MUTEX(cpu_add_remove_lock); 107 bool cpuhp_tasks_frozen; 108 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen); 109 110 /* 111 * The following two APIs (cpu_maps_update_begin/done) must be used when 112 * attempting to serialize the updates to cpu_online_mask & cpu_present_mask. 113 * The APIs cpu_notifier_register_begin/done() must be used to protect CPU 114 * hotplug callback (un)registration performed using __register_cpu_notifier() 115 * or __unregister_cpu_notifier(). 116 */ 117 void cpu_maps_update_begin(void) 118 { 119 mutex_lock(&cpu_add_remove_lock); 120 } 121 EXPORT_SYMBOL(cpu_notifier_register_begin); 122 123 void cpu_maps_update_done(void) 124 { 125 mutex_unlock(&cpu_add_remove_lock); 126 } 127 EXPORT_SYMBOL(cpu_notifier_register_done); 128 129 static RAW_NOTIFIER_HEAD(cpu_chain); 130 131 /* If set, cpu_up and cpu_down will return -EBUSY and do nothing. 132 * Should always be manipulated under cpu_add_remove_lock 133 */ 134 static int cpu_hotplug_disabled; 135 136 #ifdef CONFIG_HOTPLUG_CPU 137 138 static struct { 139 struct task_struct *active_writer; 140 /* wait queue to wake up the active_writer */ 141 wait_queue_head_t wq; 142 /* verifies that no writer will get active while readers are active */ 143 struct mutex lock; 144 /* 145 * Also blocks the new readers during 146 * an ongoing cpu hotplug operation. 147 */ 148 atomic_t refcount; 149 150 #ifdef CONFIG_DEBUG_LOCK_ALLOC 151 struct lockdep_map dep_map; 152 #endif 153 } cpu_hotplug = { 154 .active_writer = NULL, 155 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), 156 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), 157 #ifdef CONFIG_DEBUG_LOCK_ALLOC 158 .dep_map = {.name = "cpu_hotplug.lock" }, 159 #endif 160 }; 161 162 /* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */ 163 #define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map) 164 #define cpuhp_lock_acquire_tryread() \ 165 lock_map_acquire_tryread(&cpu_hotplug.dep_map) 166 #define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) 167 #define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) 168 169 170 void get_online_cpus(void) 171 { 172 might_sleep(); 173 if (cpu_hotplug.active_writer == current) 174 return; 175 cpuhp_lock_acquire_read(); 176 mutex_lock(&cpu_hotplug.lock); 177 atomic_inc(&cpu_hotplug.refcount); 178 mutex_unlock(&cpu_hotplug.lock); 179 } 180 EXPORT_SYMBOL_GPL(get_online_cpus); 181 182 void put_online_cpus(void) 183 { 184 int refcount; 185 186 if (cpu_hotplug.active_writer == current) 187 return; 188 189 refcount = atomic_dec_return(&cpu_hotplug.refcount); 190 if (WARN_ON(refcount < 0)) /* try to fix things up */ 191 atomic_inc(&cpu_hotplug.refcount); 192 193 if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq)) 194 wake_up(&cpu_hotplug.wq); 195 196 cpuhp_lock_release(); 197 198 } 199 EXPORT_SYMBOL_GPL(put_online_cpus); 200 201 /* 202 * This ensures that the hotplug operation can begin only when the 203 * refcount goes to zero. 204 * 205 * Note that during a cpu-hotplug operation, the new readers, if any, 206 * will be blocked by the cpu_hotplug.lock 207 * 208 * Since cpu_hotplug_begin() is always called after invoking 209 * cpu_maps_update_begin(), we can be sure that only one writer is active. 210 * 211 * Note that theoretically, there is a possibility of a livelock: 212 * - Refcount goes to zero, last reader wakes up the sleeping 213 * writer. 214 * - Last reader unlocks the cpu_hotplug.lock. 215 * - A new reader arrives at this moment, bumps up the refcount. 216 * - The writer acquires the cpu_hotplug.lock finds the refcount 217 * non zero and goes to sleep again. 218 * 219 * However, this is very difficult to achieve in practice since 220 * get_online_cpus() not an api which is called all that often. 221 * 222 */ 223 void cpu_hotplug_begin(void) 224 { 225 DEFINE_WAIT(wait); 226 227 cpu_hotplug.active_writer = current; 228 cpuhp_lock_acquire(); 229 230 for (;;) { 231 mutex_lock(&cpu_hotplug.lock); 232 prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE); 233 if (likely(!atomic_read(&cpu_hotplug.refcount))) 234 break; 235 mutex_unlock(&cpu_hotplug.lock); 236 schedule(); 237 } 238 finish_wait(&cpu_hotplug.wq, &wait); 239 } 240 241 void cpu_hotplug_done(void) 242 { 243 cpu_hotplug.active_writer = NULL; 244 mutex_unlock(&cpu_hotplug.lock); 245 cpuhp_lock_release(); 246 } 247 248 /* 249 * Wait for currently running CPU hotplug operations to complete (if any) and 250 * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects 251 * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the 252 * hotplug path before performing hotplug operations. So acquiring that lock 253 * guarantees mutual exclusion from any currently running hotplug operations. 254 */ 255 void cpu_hotplug_disable(void) 256 { 257 cpu_maps_update_begin(); 258 cpu_hotplug_disabled++; 259 cpu_maps_update_done(); 260 } 261 EXPORT_SYMBOL_GPL(cpu_hotplug_disable); 262 263 void cpu_hotplug_enable(void) 264 { 265 cpu_maps_update_begin(); 266 WARN_ON(--cpu_hotplug_disabled < 0); 267 cpu_maps_update_done(); 268 } 269 EXPORT_SYMBOL_GPL(cpu_hotplug_enable); 270 #endif /* CONFIG_HOTPLUG_CPU */ 271 272 /* Need to know about CPUs going up/down? */ 273 int register_cpu_notifier(struct notifier_block *nb) 274 { 275 int ret; 276 cpu_maps_update_begin(); 277 ret = raw_notifier_chain_register(&cpu_chain, nb); 278 cpu_maps_update_done(); 279 return ret; 280 } 281 282 int __register_cpu_notifier(struct notifier_block *nb) 283 { 284 return raw_notifier_chain_register(&cpu_chain, nb); 285 } 286 287 static int __cpu_notify(unsigned long val, unsigned int cpu, int nr_to_call, 288 int *nr_calls) 289 { 290 unsigned long mod = cpuhp_tasks_frozen ? CPU_TASKS_FROZEN : 0; 291 void *hcpu = (void *)(long)cpu; 292 293 int ret; 294 295 ret = __raw_notifier_call_chain(&cpu_chain, val | mod, hcpu, nr_to_call, 296 nr_calls); 297 298 return notifier_to_errno(ret); 299 } 300 301 static int cpu_notify(unsigned long val, unsigned int cpu) 302 { 303 return __cpu_notify(val, cpu, -1, NULL); 304 } 305 306 static void cpu_notify_nofail(unsigned long val, unsigned int cpu) 307 { 308 BUG_ON(cpu_notify(val, cpu)); 309 } 310 311 /* Notifier wrappers for transitioning to state machine */ 312 static int notify_prepare(unsigned int cpu) 313 { 314 int nr_calls = 0; 315 int ret; 316 317 ret = __cpu_notify(CPU_UP_PREPARE, cpu, -1, &nr_calls); 318 if (ret) { 319 nr_calls--; 320 printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n", 321 __func__, cpu); 322 __cpu_notify(CPU_UP_CANCELED, cpu, nr_calls, NULL); 323 } 324 return ret; 325 } 326 327 static int notify_online(unsigned int cpu) 328 { 329 cpu_notify(CPU_ONLINE, cpu); 330 return 0; 331 } 332 333 static int notify_starting(unsigned int cpu) 334 { 335 cpu_notify(CPU_STARTING, cpu); 336 return 0; 337 } 338 339 static int bringup_wait_for_ap(unsigned int cpu) 340 { 341 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 342 343 wait_for_completion(&st->done); 344 return st->result; 345 } 346 347 static int bringup_cpu(unsigned int cpu) 348 { 349 struct task_struct *idle = idle_thread_get(cpu); 350 int ret; 351 352 /* Arch-specific enabling code. */ 353 ret = __cpu_up(cpu, idle); 354 if (ret) { 355 cpu_notify(CPU_UP_CANCELED, cpu); 356 return ret; 357 } 358 ret = bringup_wait_for_ap(cpu); 359 BUG_ON(!cpu_online(cpu)); 360 return ret; 361 } 362 363 /* 364 * Hotplug state machine related functions 365 */ 366 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st, 367 struct cpuhp_step *steps) 368 { 369 for (st->state++; st->state < st->target; st->state++) { 370 struct cpuhp_step *step = steps + st->state; 371 372 if (!step->skip_onerr) 373 cpuhp_invoke_callback(cpu, st->state, step->startup); 374 } 375 } 376 377 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, 378 struct cpuhp_step *steps, enum cpuhp_state target) 379 { 380 enum cpuhp_state prev_state = st->state; 381 int ret = 0; 382 383 for (; st->state > target; st->state--) { 384 struct cpuhp_step *step = steps + st->state; 385 386 ret = cpuhp_invoke_callback(cpu, st->state, step->teardown); 387 if (ret) { 388 st->target = prev_state; 389 undo_cpu_down(cpu, st, steps); 390 break; 391 } 392 } 393 return ret; 394 } 395 396 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st, 397 struct cpuhp_step *steps) 398 { 399 for (st->state--; st->state > st->target; st->state--) { 400 struct cpuhp_step *step = steps + st->state; 401 402 if (!step->skip_onerr) 403 cpuhp_invoke_callback(cpu, st->state, step->teardown); 404 } 405 } 406 407 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st, 408 struct cpuhp_step *steps, enum cpuhp_state target) 409 { 410 enum cpuhp_state prev_state = st->state; 411 int ret = 0; 412 413 while (st->state < target) { 414 struct cpuhp_step *step; 415 416 st->state++; 417 step = steps + st->state; 418 ret = cpuhp_invoke_callback(cpu, st->state, step->startup); 419 if (ret) { 420 st->target = prev_state; 421 undo_cpu_up(cpu, st, steps); 422 break; 423 } 424 } 425 return ret; 426 } 427 428 /* 429 * The cpu hotplug threads manage the bringup and teardown of the cpus 430 */ 431 static void cpuhp_create(unsigned int cpu) 432 { 433 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 434 435 init_completion(&st->done); 436 } 437 438 static int cpuhp_should_run(unsigned int cpu) 439 { 440 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 441 442 return st->should_run; 443 } 444 445 /* Execute the teardown callbacks. Used to be CPU_DOWN_PREPARE */ 446 static int cpuhp_ap_offline(unsigned int cpu, struct cpuhp_cpu_state *st) 447 { 448 enum cpuhp_state target = max((int)st->target, CPUHP_TEARDOWN_CPU); 449 450 return cpuhp_down_callbacks(cpu, st, cpuhp_ap_states, target); 451 } 452 453 /* Execute the online startup callbacks. Used to be CPU_ONLINE */ 454 static int cpuhp_ap_online(unsigned int cpu, struct cpuhp_cpu_state *st) 455 { 456 return cpuhp_up_callbacks(cpu, st, cpuhp_ap_states, st->target); 457 } 458 459 /* 460 * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke 461 * callbacks when a state gets [un]installed at runtime. 462 */ 463 static void cpuhp_thread_fun(unsigned int cpu) 464 { 465 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 466 int ret = 0; 467 468 /* 469 * Paired with the mb() in cpuhp_kick_ap_work and 470 * cpuhp_invoke_ap_callback, so the work set is consistent visible. 471 */ 472 smp_mb(); 473 if (!st->should_run) 474 return; 475 476 st->should_run = false; 477 478 /* Single callback invocation for [un]install ? */ 479 if (st->cb) { 480 if (st->cb_state < CPUHP_AP_ONLINE) { 481 local_irq_disable(); 482 ret = cpuhp_invoke_callback(cpu, st->cb_state, st->cb); 483 local_irq_enable(); 484 } else { 485 ret = cpuhp_invoke_callback(cpu, st->cb_state, st->cb); 486 } 487 } else if (st->rollback) { 488 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE); 489 490 undo_cpu_down(cpu, st, cpuhp_ap_states); 491 /* 492 * This is a momentary workaround to keep the notifier users 493 * happy. Will go away once we got rid of the notifiers. 494 */ 495 cpu_notify_nofail(CPU_DOWN_FAILED, cpu); 496 st->rollback = false; 497 } else { 498 /* Cannot happen .... */ 499 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE); 500 501 /* Regular hotplug work */ 502 if (st->state < st->target) 503 ret = cpuhp_ap_online(cpu, st); 504 else if (st->state > st->target) 505 ret = cpuhp_ap_offline(cpu, st); 506 } 507 st->result = ret; 508 complete(&st->done); 509 } 510 511 /* Invoke a single callback on a remote cpu */ 512 static int cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, 513 int (*cb)(unsigned int)) 514 { 515 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 516 517 if (!cpu_online(cpu)) 518 return 0; 519 520 st->cb_state = state; 521 st->cb = cb; 522 /* 523 * Make sure the above stores are visible before should_run becomes 524 * true. Paired with the mb() above in cpuhp_thread_fun() 525 */ 526 smp_mb(); 527 st->should_run = true; 528 wake_up_process(st->thread); 529 wait_for_completion(&st->done); 530 return st->result; 531 } 532 533 /* Regular hotplug invocation of the AP hotplug thread */ 534 static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st) 535 { 536 st->result = 0; 537 st->cb = NULL; 538 /* 539 * Make sure the above stores are visible before should_run becomes 540 * true. Paired with the mb() above in cpuhp_thread_fun() 541 */ 542 smp_mb(); 543 st->should_run = true; 544 wake_up_process(st->thread); 545 } 546 547 static int cpuhp_kick_ap_work(unsigned int cpu) 548 { 549 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 550 enum cpuhp_state state = st->state; 551 552 trace_cpuhp_enter(cpu, st->target, state, cpuhp_kick_ap_work); 553 __cpuhp_kick_ap_work(st); 554 wait_for_completion(&st->done); 555 trace_cpuhp_exit(cpu, st->state, state, st->result); 556 return st->result; 557 } 558 559 static struct smp_hotplug_thread cpuhp_threads = { 560 .store = &cpuhp_state.thread, 561 .create = &cpuhp_create, 562 .thread_should_run = cpuhp_should_run, 563 .thread_fn = cpuhp_thread_fun, 564 .thread_comm = "cpuhp/%u", 565 .selfparking = true, 566 }; 567 568 void __init cpuhp_threads_init(void) 569 { 570 BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads)); 571 kthread_unpark(this_cpu_read(cpuhp_state.thread)); 572 } 573 574 #ifdef CONFIG_HOTPLUG_CPU 575 EXPORT_SYMBOL(register_cpu_notifier); 576 EXPORT_SYMBOL(__register_cpu_notifier); 577 void unregister_cpu_notifier(struct notifier_block *nb) 578 { 579 cpu_maps_update_begin(); 580 raw_notifier_chain_unregister(&cpu_chain, nb); 581 cpu_maps_update_done(); 582 } 583 EXPORT_SYMBOL(unregister_cpu_notifier); 584 585 void __unregister_cpu_notifier(struct notifier_block *nb) 586 { 587 raw_notifier_chain_unregister(&cpu_chain, nb); 588 } 589 EXPORT_SYMBOL(__unregister_cpu_notifier); 590 591 /** 592 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU 593 * @cpu: a CPU id 594 * 595 * This function walks all processes, finds a valid mm struct for each one and 596 * then clears a corresponding bit in mm's cpumask. While this all sounds 597 * trivial, there are various non-obvious corner cases, which this function 598 * tries to solve in a safe manner. 599 * 600 * Also note that the function uses a somewhat relaxed locking scheme, so it may 601 * be called only for an already offlined CPU. 602 */ 603 void clear_tasks_mm_cpumask(int cpu) 604 { 605 struct task_struct *p; 606 607 /* 608 * This function is called after the cpu is taken down and marked 609 * offline, so its not like new tasks will ever get this cpu set in 610 * their mm mask. -- Peter Zijlstra 611 * Thus, we may use rcu_read_lock() here, instead of grabbing 612 * full-fledged tasklist_lock. 613 */ 614 WARN_ON(cpu_online(cpu)); 615 rcu_read_lock(); 616 for_each_process(p) { 617 struct task_struct *t; 618 619 /* 620 * Main thread might exit, but other threads may still have 621 * a valid mm. Find one. 622 */ 623 t = find_lock_task_mm(p); 624 if (!t) 625 continue; 626 cpumask_clear_cpu(cpu, mm_cpumask(t->mm)); 627 task_unlock(t); 628 } 629 rcu_read_unlock(); 630 } 631 632 static inline void check_for_tasks(int dead_cpu) 633 { 634 struct task_struct *g, *p; 635 636 read_lock(&tasklist_lock); 637 for_each_process_thread(g, p) { 638 if (!p->on_rq) 639 continue; 640 /* 641 * We do the check with unlocked task_rq(p)->lock. 642 * Order the reading to do not warn about a task, 643 * which was running on this cpu in the past, and 644 * it's just been woken on another cpu. 645 */ 646 rmb(); 647 if (task_cpu(p) != dead_cpu) 648 continue; 649 650 pr_warn("Task %s (pid=%d) is on cpu %d (state=%ld, flags=%x)\n", 651 p->comm, task_pid_nr(p), dead_cpu, p->state, p->flags); 652 } 653 read_unlock(&tasklist_lock); 654 } 655 656 static int notify_down_prepare(unsigned int cpu) 657 { 658 int err, nr_calls = 0; 659 660 err = __cpu_notify(CPU_DOWN_PREPARE, cpu, -1, &nr_calls); 661 if (err) { 662 nr_calls--; 663 __cpu_notify(CPU_DOWN_FAILED, cpu, nr_calls, NULL); 664 pr_warn("%s: attempt to take down CPU %u failed\n", 665 __func__, cpu); 666 } 667 return err; 668 } 669 670 static int notify_dying(unsigned int cpu) 671 { 672 cpu_notify(CPU_DYING, cpu); 673 return 0; 674 } 675 676 /* Take this CPU down. */ 677 static int take_cpu_down(void *_param) 678 { 679 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 680 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); 681 int err, cpu = smp_processor_id(); 682 683 /* Ensure this CPU doesn't handle any more interrupts. */ 684 err = __cpu_disable(); 685 if (err < 0) 686 return err; 687 688 /* Invoke the former CPU_DYING callbacks */ 689 for (; st->state > target; st->state--) { 690 struct cpuhp_step *step = cpuhp_ap_states + st->state; 691 692 cpuhp_invoke_callback(cpu, st->state, step->teardown); 693 } 694 /* Give up timekeeping duties */ 695 tick_handover_do_timer(); 696 /* Park the stopper thread */ 697 stop_machine_park(cpu); 698 return 0; 699 } 700 701 static int takedown_cpu(unsigned int cpu) 702 { 703 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 704 int err; 705 706 /* Park the smpboot threads */ 707 kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread); 708 smpboot_park_threads(cpu); 709 710 /* 711 * Prevent irq alloc/free while the dying cpu reorganizes the 712 * interrupt affinities. 713 */ 714 irq_lock_sparse(); 715 716 /* 717 * So now all preempt/rcu users must observe !cpu_active(). 718 */ 719 err = stop_machine(take_cpu_down, NULL, cpumask_of(cpu)); 720 if (err) { 721 /* CPU refused to die */ 722 irq_unlock_sparse(); 723 /* Unpark the hotplug thread so we can rollback there */ 724 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread); 725 return err; 726 } 727 BUG_ON(cpu_online(cpu)); 728 729 /* 730 * The migration_call() CPU_DYING callback will have removed all 731 * runnable tasks from the cpu, there's only the idle task left now 732 * that the migration thread is done doing the stop_machine thing. 733 * 734 * Wait for the stop thread to go away. 735 */ 736 wait_for_completion(&st->done); 737 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); 738 739 /* Interrupts are moved away from the dying cpu, reenable alloc/free */ 740 irq_unlock_sparse(); 741 742 hotplug_cpu__broadcast_tick_pull(cpu); 743 /* This actually kills the CPU. */ 744 __cpu_die(cpu); 745 746 tick_cleanup_dead_cpu(cpu); 747 return 0; 748 } 749 750 static int notify_dead(unsigned int cpu) 751 { 752 cpu_notify_nofail(CPU_DEAD, cpu); 753 check_for_tasks(cpu); 754 return 0; 755 } 756 757 static void cpuhp_complete_idle_dead(void *arg) 758 { 759 struct cpuhp_cpu_state *st = arg; 760 761 complete(&st->done); 762 } 763 764 void cpuhp_report_idle_dead(void) 765 { 766 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 767 768 BUG_ON(st->state != CPUHP_AP_OFFLINE); 769 rcu_report_dead(smp_processor_id()); 770 st->state = CPUHP_AP_IDLE_DEAD; 771 /* 772 * We cannot call complete after rcu_report_dead() so we delegate it 773 * to an online cpu. 774 */ 775 smp_call_function_single(cpumask_first(cpu_online_mask), 776 cpuhp_complete_idle_dead, st, 0); 777 } 778 779 #else 780 #define notify_down_prepare NULL 781 #define takedown_cpu NULL 782 #define notify_dead NULL 783 #define notify_dying NULL 784 #endif 785 786 #ifdef CONFIG_HOTPLUG_CPU 787 788 /* Requires cpu_add_remove_lock to be held */ 789 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen, 790 enum cpuhp_state target) 791 { 792 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 793 int prev_state, ret = 0; 794 bool hasdied = false; 795 796 if (num_online_cpus() == 1) 797 return -EBUSY; 798 799 if (!cpu_present(cpu)) 800 return -EINVAL; 801 802 cpu_hotplug_begin(); 803 804 cpuhp_tasks_frozen = tasks_frozen; 805 806 prev_state = st->state; 807 st->target = target; 808 /* 809 * If the current CPU state is in the range of the AP hotplug thread, 810 * then we need to kick the thread. 811 */ 812 if (st->state > CPUHP_TEARDOWN_CPU) { 813 ret = cpuhp_kick_ap_work(cpu); 814 /* 815 * The AP side has done the error rollback already. Just 816 * return the error code.. 817 */ 818 if (ret) 819 goto out; 820 821 /* 822 * We might have stopped still in the range of the AP hotplug 823 * thread. Nothing to do anymore. 824 */ 825 if (st->state > CPUHP_TEARDOWN_CPU) 826 goto out; 827 } 828 /* 829 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need 830 * to do the further cleanups. 831 */ 832 ret = cpuhp_down_callbacks(cpu, st, cpuhp_bp_states, target); 833 if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) { 834 st->target = prev_state; 835 st->rollback = true; 836 cpuhp_kick_ap_work(cpu); 837 } 838 839 hasdied = prev_state != st->state && st->state == CPUHP_OFFLINE; 840 out: 841 cpu_hotplug_done(); 842 /* This post dead nonsense must die */ 843 if (!ret && hasdied) 844 cpu_notify_nofail(CPU_POST_DEAD, cpu); 845 return ret; 846 } 847 848 static int do_cpu_down(unsigned int cpu, enum cpuhp_state target) 849 { 850 int err; 851 852 cpu_maps_update_begin(); 853 854 if (cpu_hotplug_disabled) { 855 err = -EBUSY; 856 goto out; 857 } 858 859 err = _cpu_down(cpu, 0, target); 860 861 out: 862 cpu_maps_update_done(); 863 return err; 864 } 865 int cpu_down(unsigned int cpu) 866 { 867 return do_cpu_down(cpu, CPUHP_OFFLINE); 868 } 869 EXPORT_SYMBOL(cpu_down); 870 #endif /*CONFIG_HOTPLUG_CPU*/ 871 872 /** 873 * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers 874 * @cpu: cpu that just started 875 * 876 * This function calls the cpu_chain notifiers with CPU_STARTING. 877 * It must be called by the arch code on the new cpu, before the new cpu 878 * enables interrupts and before the "boot" cpu returns from __cpu_up(). 879 */ 880 void notify_cpu_starting(unsigned int cpu) 881 { 882 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 883 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); 884 885 while (st->state < target) { 886 struct cpuhp_step *step; 887 888 st->state++; 889 step = cpuhp_ap_states + st->state; 890 cpuhp_invoke_callback(cpu, st->state, step->startup); 891 } 892 } 893 894 /* 895 * Called from the idle task. We need to set active here, so we can kick off 896 * the stopper thread and unpark the smpboot threads. If the target state is 897 * beyond CPUHP_AP_ONLINE_IDLE we kick cpuhp thread and let it bring up the 898 * cpu further. 899 */ 900 void cpuhp_online_idle(enum cpuhp_state state) 901 { 902 struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state); 903 unsigned int cpu = smp_processor_id(); 904 905 /* Happens for the boot cpu */ 906 if (state != CPUHP_AP_ONLINE_IDLE) 907 return; 908 909 st->state = CPUHP_AP_ONLINE_IDLE; 910 911 /* Unpark the stopper thread and the hotplug thread of this cpu */ 912 stop_machine_unpark(cpu); 913 kthread_unpark(st->thread); 914 915 /* Should we go further up ? */ 916 if (st->target > CPUHP_AP_ONLINE_IDLE) 917 __cpuhp_kick_ap_work(st); 918 else 919 complete(&st->done); 920 } 921 922 /* Requires cpu_add_remove_lock to be held */ 923 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target) 924 { 925 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 926 struct task_struct *idle; 927 int ret = 0; 928 929 cpu_hotplug_begin(); 930 931 if (!cpu_present(cpu)) { 932 ret = -EINVAL; 933 goto out; 934 } 935 936 /* 937 * The caller of do_cpu_up might have raced with another 938 * caller. Ignore it for now. 939 */ 940 if (st->state >= target) 941 goto out; 942 943 if (st->state == CPUHP_OFFLINE) { 944 /* Let it fail before we try to bring the cpu up */ 945 idle = idle_thread_get(cpu); 946 if (IS_ERR(idle)) { 947 ret = PTR_ERR(idle); 948 goto out; 949 } 950 } 951 952 cpuhp_tasks_frozen = tasks_frozen; 953 954 st->target = target; 955 /* 956 * If the current CPU state is in the range of the AP hotplug thread, 957 * then we need to kick the thread once more. 958 */ 959 if (st->state > CPUHP_BRINGUP_CPU) { 960 ret = cpuhp_kick_ap_work(cpu); 961 /* 962 * The AP side has done the error rollback already. Just 963 * return the error code.. 964 */ 965 if (ret) 966 goto out; 967 } 968 969 /* 970 * Try to reach the target state. We max out on the BP at 971 * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is 972 * responsible for bringing it up to the target state. 973 */ 974 target = min((int)target, CPUHP_BRINGUP_CPU); 975 ret = cpuhp_up_callbacks(cpu, st, cpuhp_bp_states, target); 976 out: 977 cpu_hotplug_done(); 978 return ret; 979 } 980 981 static int do_cpu_up(unsigned int cpu, enum cpuhp_state target) 982 { 983 int err = 0; 984 985 if (!cpu_possible(cpu)) { 986 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n", 987 cpu); 988 #if defined(CONFIG_IA64) 989 pr_err("please check additional_cpus= boot parameter\n"); 990 #endif 991 return -EINVAL; 992 } 993 994 err = try_online_node(cpu_to_node(cpu)); 995 if (err) 996 return err; 997 998 cpu_maps_update_begin(); 999 1000 if (cpu_hotplug_disabled) { 1001 err = -EBUSY; 1002 goto out; 1003 } 1004 1005 err = _cpu_up(cpu, 0, target); 1006 out: 1007 cpu_maps_update_done(); 1008 return err; 1009 } 1010 1011 int cpu_up(unsigned int cpu) 1012 { 1013 return do_cpu_up(cpu, CPUHP_ONLINE); 1014 } 1015 EXPORT_SYMBOL_GPL(cpu_up); 1016 1017 #ifdef CONFIG_PM_SLEEP_SMP 1018 static cpumask_var_t frozen_cpus; 1019 1020 int disable_nonboot_cpus(void) 1021 { 1022 int cpu, first_cpu, error = 0; 1023 1024 cpu_maps_update_begin(); 1025 first_cpu = cpumask_first(cpu_online_mask); 1026 /* 1027 * We take down all of the non-boot CPUs in one shot to avoid races 1028 * with the userspace trying to use the CPU hotplug at the same time 1029 */ 1030 cpumask_clear(frozen_cpus); 1031 1032 pr_info("Disabling non-boot CPUs ...\n"); 1033 for_each_online_cpu(cpu) { 1034 if (cpu == first_cpu) 1035 continue; 1036 trace_suspend_resume(TPS("CPU_OFF"), cpu, true); 1037 error = _cpu_down(cpu, 1, CPUHP_OFFLINE); 1038 trace_suspend_resume(TPS("CPU_OFF"), cpu, false); 1039 if (!error) 1040 cpumask_set_cpu(cpu, frozen_cpus); 1041 else { 1042 pr_err("Error taking CPU%d down: %d\n", cpu, error); 1043 break; 1044 } 1045 } 1046 1047 if (!error) 1048 BUG_ON(num_online_cpus() > 1); 1049 else 1050 pr_err("Non-boot CPUs are not disabled\n"); 1051 1052 /* 1053 * Make sure the CPUs won't be enabled by someone else. We need to do 1054 * this even in case of failure as all disable_nonboot_cpus() users are 1055 * supposed to do enable_nonboot_cpus() on the failure path. 1056 */ 1057 cpu_hotplug_disabled++; 1058 1059 cpu_maps_update_done(); 1060 return error; 1061 } 1062 1063 void __weak arch_enable_nonboot_cpus_begin(void) 1064 { 1065 } 1066 1067 void __weak arch_enable_nonboot_cpus_end(void) 1068 { 1069 } 1070 1071 void enable_nonboot_cpus(void) 1072 { 1073 int cpu, error; 1074 1075 /* Allow everyone to use the CPU hotplug again */ 1076 cpu_maps_update_begin(); 1077 WARN_ON(--cpu_hotplug_disabled < 0); 1078 if (cpumask_empty(frozen_cpus)) 1079 goto out; 1080 1081 pr_info("Enabling non-boot CPUs ...\n"); 1082 1083 arch_enable_nonboot_cpus_begin(); 1084 1085 for_each_cpu(cpu, frozen_cpus) { 1086 trace_suspend_resume(TPS("CPU_ON"), cpu, true); 1087 error = _cpu_up(cpu, 1, CPUHP_ONLINE); 1088 trace_suspend_resume(TPS("CPU_ON"), cpu, false); 1089 if (!error) { 1090 pr_info("CPU%d is up\n", cpu); 1091 continue; 1092 } 1093 pr_warn("Error taking CPU%d up: %d\n", cpu, error); 1094 } 1095 1096 arch_enable_nonboot_cpus_end(); 1097 1098 cpumask_clear(frozen_cpus); 1099 out: 1100 cpu_maps_update_done(); 1101 } 1102 1103 static int __init alloc_frozen_cpus(void) 1104 { 1105 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO)) 1106 return -ENOMEM; 1107 return 0; 1108 } 1109 core_initcall(alloc_frozen_cpus); 1110 1111 /* 1112 * When callbacks for CPU hotplug notifications are being executed, we must 1113 * ensure that the state of the system with respect to the tasks being frozen 1114 * or not, as reported by the notification, remains unchanged *throughout the 1115 * duration* of the execution of the callbacks. 1116 * Hence we need to prevent the freezer from racing with regular CPU hotplug. 1117 * 1118 * This synchronization is implemented by mutually excluding regular CPU 1119 * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/ 1120 * Hibernate notifications. 1121 */ 1122 static int 1123 cpu_hotplug_pm_callback(struct notifier_block *nb, 1124 unsigned long action, void *ptr) 1125 { 1126 switch (action) { 1127 1128 case PM_SUSPEND_PREPARE: 1129 case PM_HIBERNATION_PREPARE: 1130 cpu_hotplug_disable(); 1131 break; 1132 1133 case PM_POST_SUSPEND: 1134 case PM_POST_HIBERNATION: 1135 cpu_hotplug_enable(); 1136 break; 1137 1138 default: 1139 return NOTIFY_DONE; 1140 } 1141 1142 return NOTIFY_OK; 1143 } 1144 1145 1146 static int __init cpu_hotplug_pm_sync_init(void) 1147 { 1148 /* 1149 * cpu_hotplug_pm_callback has higher priority than x86 1150 * bsp_pm_callback which depends on cpu_hotplug_pm_callback 1151 * to disable cpu hotplug to avoid cpu hotplug race. 1152 */ 1153 pm_notifier(cpu_hotplug_pm_callback, 0); 1154 return 0; 1155 } 1156 core_initcall(cpu_hotplug_pm_sync_init); 1157 1158 #endif /* CONFIG_PM_SLEEP_SMP */ 1159 1160 #endif /* CONFIG_SMP */ 1161 1162 /* Boot processor state steps */ 1163 static struct cpuhp_step cpuhp_bp_states[] = { 1164 [CPUHP_OFFLINE] = { 1165 .name = "offline", 1166 .startup = NULL, 1167 .teardown = NULL, 1168 }, 1169 #ifdef CONFIG_SMP 1170 [CPUHP_CREATE_THREADS]= { 1171 .name = "threads:create", 1172 .startup = smpboot_create_threads, 1173 .teardown = NULL, 1174 .cant_stop = true, 1175 }, 1176 /* 1177 * Preparatory and dead notifiers. Will be replaced once the notifiers 1178 * are converted to states. 1179 */ 1180 [CPUHP_NOTIFY_PREPARE] = { 1181 .name = "notify:prepare", 1182 .startup = notify_prepare, 1183 .teardown = notify_dead, 1184 .skip_onerr = true, 1185 .cant_stop = true, 1186 }, 1187 /* Kicks the plugged cpu into life */ 1188 [CPUHP_BRINGUP_CPU] = { 1189 .name = "cpu:bringup", 1190 .startup = bringup_cpu, 1191 .teardown = NULL, 1192 .cant_stop = true, 1193 }, 1194 /* 1195 * Handled on controll processor until the plugged processor manages 1196 * this itself. 1197 */ 1198 [CPUHP_TEARDOWN_CPU] = { 1199 .name = "cpu:teardown", 1200 .startup = NULL, 1201 .teardown = takedown_cpu, 1202 .cant_stop = true, 1203 }, 1204 #else 1205 [CPUHP_BRINGUP_CPU] = { }, 1206 #endif 1207 }; 1208 1209 /* Application processor state steps */ 1210 static struct cpuhp_step cpuhp_ap_states[] = { 1211 #ifdef CONFIG_SMP 1212 /* Final state before CPU kills itself */ 1213 [CPUHP_AP_IDLE_DEAD] = { 1214 .name = "idle:dead", 1215 }, 1216 /* 1217 * Last state before CPU enters the idle loop to die. Transient state 1218 * for synchronization. 1219 */ 1220 [CPUHP_AP_OFFLINE] = { 1221 .name = "ap:offline", 1222 .cant_stop = true, 1223 }, 1224 /* First state is scheduler control. Interrupts are disabled */ 1225 [CPUHP_AP_SCHED_STARTING] = { 1226 .name = "sched:starting", 1227 .startup = sched_cpu_starting, 1228 .teardown = sched_cpu_dying, 1229 }, 1230 /* 1231 * Low level startup/teardown notifiers. Run with interrupts 1232 * disabled. Will be removed once the notifiers are converted to 1233 * states. 1234 */ 1235 [CPUHP_AP_NOTIFY_STARTING] = { 1236 .name = "notify:starting", 1237 .startup = notify_starting, 1238 .teardown = notify_dying, 1239 .skip_onerr = true, 1240 .cant_stop = true, 1241 }, 1242 /* Entry state on starting. Interrupts enabled from here on. Transient 1243 * state for synchronsization */ 1244 [CPUHP_AP_ONLINE] = { 1245 .name = "ap:online", 1246 }, 1247 /* Handle smpboot threads park/unpark */ 1248 [CPUHP_AP_SMPBOOT_THREADS] = { 1249 .name = "smpboot:threads", 1250 .startup = smpboot_unpark_threads, 1251 .teardown = NULL, 1252 }, 1253 /* 1254 * Online/down_prepare notifiers. Will be removed once the notifiers 1255 * are converted to states. 1256 */ 1257 [CPUHP_AP_NOTIFY_ONLINE] = { 1258 .name = "notify:online", 1259 .startup = notify_online, 1260 .teardown = notify_down_prepare, 1261 .skip_onerr = true, 1262 }, 1263 #endif 1264 /* 1265 * The dynamically registered state space is here 1266 */ 1267 1268 #ifdef CONFIG_SMP 1269 /* Last state is scheduler control setting the cpu active */ 1270 [CPUHP_AP_ACTIVE] = { 1271 .name = "sched:active", 1272 .startup = sched_cpu_activate, 1273 .teardown = sched_cpu_deactivate, 1274 }, 1275 #endif 1276 1277 /* CPU is fully up and running. */ 1278 [CPUHP_ONLINE] = { 1279 .name = "online", 1280 .startup = NULL, 1281 .teardown = NULL, 1282 }, 1283 }; 1284 1285 /* Sanity check for callbacks */ 1286 static int cpuhp_cb_check(enum cpuhp_state state) 1287 { 1288 if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE) 1289 return -EINVAL; 1290 return 0; 1291 } 1292 1293 static bool cpuhp_is_ap_state(enum cpuhp_state state) 1294 { 1295 /* 1296 * The extra check for CPUHP_TEARDOWN_CPU is only for documentation 1297 * purposes as that state is handled explicitely in cpu_down. 1298 */ 1299 return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU; 1300 } 1301 1302 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state) 1303 { 1304 struct cpuhp_step *sp; 1305 1306 sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states; 1307 return sp + state; 1308 } 1309 1310 static void cpuhp_store_callbacks(enum cpuhp_state state, 1311 const char *name, 1312 int (*startup)(unsigned int cpu), 1313 int (*teardown)(unsigned int cpu)) 1314 { 1315 /* (Un)Install the callbacks for further cpu hotplug operations */ 1316 struct cpuhp_step *sp; 1317 1318 mutex_lock(&cpuhp_state_mutex); 1319 sp = cpuhp_get_step(state); 1320 sp->startup = startup; 1321 sp->teardown = teardown; 1322 sp->name = name; 1323 mutex_unlock(&cpuhp_state_mutex); 1324 } 1325 1326 static void *cpuhp_get_teardown_cb(enum cpuhp_state state) 1327 { 1328 return cpuhp_get_step(state)->teardown; 1329 } 1330 1331 /* 1332 * Call the startup/teardown function for a step either on the AP or 1333 * on the current CPU. 1334 */ 1335 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, 1336 int (*cb)(unsigned int), bool bringup) 1337 { 1338 int ret; 1339 1340 if (!cb) 1341 return 0; 1342 /* 1343 * The non AP bound callbacks can fail on bringup. On teardown 1344 * e.g. module removal we crash for now. 1345 */ 1346 #ifdef CONFIG_SMP 1347 if (cpuhp_is_ap_state(state)) 1348 ret = cpuhp_invoke_ap_callback(cpu, state, cb); 1349 else 1350 ret = cpuhp_invoke_callback(cpu, state, cb); 1351 #else 1352 ret = cpuhp_invoke_callback(cpu, state, cb); 1353 #endif 1354 BUG_ON(ret && !bringup); 1355 return ret; 1356 } 1357 1358 /* 1359 * Called from __cpuhp_setup_state on a recoverable failure. 1360 * 1361 * Note: The teardown callbacks for rollback are not allowed to fail! 1362 */ 1363 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state, 1364 int (*teardown)(unsigned int cpu)) 1365 { 1366 int cpu; 1367 1368 if (!teardown) 1369 return; 1370 1371 /* Roll back the already executed steps on the other cpus */ 1372 for_each_present_cpu(cpu) { 1373 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1374 int cpustate = st->state; 1375 1376 if (cpu >= failedcpu) 1377 break; 1378 1379 /* Did we invoke the startup call on that cpu ? */ 1380 if (cpustate >= state) 1381 cpuhp_issue_call(cpu, state, teardown, false); 1382 } 1383 } 1384 1385 /* 1386 * Returns a free for dynamic slot assignment of the Online state. The states 1387 * are protected by the cpuhp_slot_states mutex and an empty slot is identified 1388 * by having no name assigned. 1389 */ 1390 static int cpuhp_reserve_state(enum cpuhp_state state) 1391 { 1392 enum cpuhp_state i; 1393 1394 mutex_lock(&cpuhp_state_mutex); 1395 for (i = CPUHP_AP_ONLINE_DYN; i <= CPUHP_AP_ONLINE_DYN_END; i++) { 1396 if (cpuhp_ap_states[i].name) 1397 continue; 1398 1399 cpuhp_ap_states[i].name = "Reserved"; 1400 mutex_unlock(&cpuhp_state_mutex); 1401 return i; 1402 } 1403 mutex_unlock(&cpuhp_state_mutex); 1404 WARN(1, "No more dynamic states available for CPU hotplug\n"); 1405 return -ENOSPC; 1406 } 1407 1408 /** 1409 * __cpuhp_setup_state - Setup the callbacks for an hotplug machine state 1410 * @state: The state to setup 1411 * @invoke: If true, the startup function is invoked for cpus where 1412 * cpu state >= @state 1413 * @startup: startup callback function 1414 * @teardown: teardown callback function 1415 * 1416 * Returns 0 if successful, otherwise a proper error code 1417 */ 1418 int __cpuhp_setup_state(enum cpuhp_state state, 1419 const char *name, bool invoke, 1420 int (*startup)(unsigned int cpu), 1421 int (*teardown)(unsigned int cpu)) 1422 { 1423 int cpu, ret = 0; 1424 int dyn_state = 0; 1425 1426 if (cpuhp_cb_check(state) || !name) 1427 return -EINVAL; 1428 1429 get_online_cpus(); 1430 1431 /* currently assignments for the ONLINE state are possible */ 1432 if (state == CPUHP_AP_ONLINE_DYN) { 1433 dyn_state = 1; 1434 ret = cpuhp_reserve_state(state); 1435 if (ret < 0) 1436 goto out; 1437 state = ret; 1438 } 1439 1440 cpuhp_store_callbacks(state, name, startup, teardown); 1441 1442 if (!invoke || !startup) 1443 goto out; 1444 1445 /* 1446 * Try to call the startup callback for each present cpu 1447 * depending on the hotplug state of the cpu. 1448 */ 1449 for_each_present_cpu(cpu) { 1450 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1451 int cpustate = st->state; 1452 1453 if (cpustate < state) 1454 continue; 1455 1456 ret = cpuhp_issue_call(cpu, state, startup, true); 1457 if (ret) { 1458 cpuhp_rollback_install(cpu, state, teardown); 1459 cpuhp_store_callbacks(state, NULL, NULL, NULL); 1460 goto out; 1461 } 1462 } 1463 out: 1464 put_online_cpus(); 1465 if (!ret && dyn_state) 1466 return state; 1467 return ret; 1468 } 1469 EXPORT_SYMBOL(__cpuhp_setup_state); 1470 1471 /** 1472 * __cpuhp_remove_state - Remove the callbacks for an hotplug machine state 1473 * @state: The state to remove 1474 * @invoke: If true, the teardown function is invoked for cpus where 1475 * cpu state >= @state 1476 * 1477 * The teardown callback is currently not allowed to fail. Think 1478 * about module removal! 1479 */ 1480 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke) 1481 { 1482 int (*teardown)(unsigned int cpu) = cpuhp_get_teardown_cb(state); 1483 int cpu; 1484 1485 BUG_ON(cpuhp_cb_check(state)); 1486 1487 get_online_cpus(); 1488 1489 if (!invoke || !teardown) 1490 goto remove; 1491 1492 /* 1493 * Call the teardown callback for each present cpu depending 1494 * on the hotplug state of the cpu. This function is not 1495 * allowed to fail currently! 1496 */ 1497 for_each_present_cpu(cpu) { 1498 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); 1499 int cpustate = st->state; 1500 1501 if (cpustate >= state) 1502 cpuhp_issue_call(cpu, state, teardown, false); 1503 } 1504 remove: 1505 cpuhp_store_callbacks(state, NULL, NULL, NULL); 1506 put_online_cpus(); 1507 } 1508 EXPORT_SYMBOL(__cpuhp_remove_state); 1509 1510 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU) 1511 static ssize_t show_cpuhp_state(struct device *dev, 1512 struct device_attribute *attr, char *buf) 1513 { 1514 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1515 1516 return sprintf(buf, "%d\n", st->state); 1517 } 1518 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL); 1519 1520 static ssize_t write_cpuhp_target(struct device *dev, 1521 struct device_attribute *attr, 1522 const char *buf, size_t count) 1523 { 1524 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1525 struct cpuhp_step *sp; 1526 int target, ret; 1527 1528 ret = kstrtoint(buf, 10, &target); 1529 if (ret) 1530 return ret; 1531 1532 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL 1533 if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE) 1534 return -EINVAL; 1535 #else 1536 if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE) 1537 return -EINVAL; 1538 #endif 1539 1540 ret = lock_device_hotplug_sysfs(); 1541 if (ret) 1542 return ret; 1543 1544 mutex_lock(&cpuhp_state_mutex); 1545 sp = cpuhp_get_step(target); 1546 ret = !sp->name || sp->cant_stop ? -EINVAL : 0; 1547 mutex_unlock(&cpuhp_state_mutex); 1548 if (ret) 1549 return ret; 1550 1551 if (st->state < target) 1552 ret = do_cpu_up(dev->id, target); 1553 else 1554 ret = do_cpu_down(dev->id, target); 1555 1556 unlock_device_hotplug(); 1557 return ret ? ret : count; 1558 } 1559 1560 static ssize_t show_cpuhp_target(struct device *dev, 1561 struct device_attribute *attr, char *buf) 1562 { 1563 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); 1564 1565 return sprintf(buf, "%d\n", st->target); 1566 } 1567 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target); 1568 1569 static struct attribute *cpuhp_cpu_attrs[] = { 1570 &dev_attr_state.attr, 1571 &dev_attr_target.attr, 1572 NULL 1573 }; 1574 1575 static struct attribute_group cpuhp_cpu_attr_group = { 1576 .attrs = cpuhp_cpu_attrs, 1577 .name = "hotplug", 1578 NULL 1579 }; 1580 1581 static ssize_t show_cpuhp_states(struct device *dev, 1582 struct device_attribute *attr, char *buf) 1583 { 1584 ssize_t cur, res = 0; 1585 int i; 1586 1587 mutex_lock(&cpuhp_state_mutex); 1588 for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) { 1589 struct cpuhp_step *sp = cpuhp_get_step(i); 1590 1591 if (sp->name) { 1592 cur = sprintf(buf, "%3d: %s\n", i, sp->name); 1593 buf += cur; 1594 res += cur; 1595 } 1596 } 1597 mutex_unlock(&cpuhp_state_mutex); 1598 return res; 1599 } 1600 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL); 1601 1602 static struct attribute *cpuhp_cpu_root_attrs[] = { 1603 &dev_attr_states.attr, 1604 NULL 1605 }; 1606 1607 static struct attribute_group cpuhp_cpu_root_attr_group = { 1608 .attrs = cpuhp_cpu_root_attrs, 1609 .name = "hotplug", 1610 NULL 1611 }; 1612 1613 static int __init cpuhp_sysfs_init(void) 1614 { 1615 int cpu, ret; 1616 1617 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, 1618 &cpuhp_cpu_root_attr_group); 1619 if (ret) 1620 return ret; 1621 1622 for_each_possible_cpu(cpu) { 1623 struct device *dev = get_cpu_device(cpu); 1624 1625 if (!dev) 1626 continue; 1627 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group); 1628 if (ret) 1629 return ret; 1630 } 1631 return 0; 1632 } 1633 device_initcall(cpuhp_sysfs_init); 1634 #endif 1635 1636 /* 1637 * cpu_bit_bitmap[] is a special, "compressed" data structure that 1638 * represents all NR_CPUS bits binary values of 1<<nr. 1639 * 1640 * It is used by cpumask_of() to get a constant address to a CPU 1641 * mask value that has a single bit set only. 1642 */ 1643 1644 /* cpu_bit_bitmap[0] is empty - so we can back into it */ 1645 #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x)) 1646 #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1) 1647 #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2) 1648 #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4) 1649 1650 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = { 1651 1652 MASK_DECLARE_8(0), MASK_DECLARE_8(8), 1653 MASK_DECLARE_8(16), MASK_DECLARE_8(24), 1654 #if BITS_PER_LONG > 32 1655 MASK_DECLARE_8(32), MASK_DECLARE_8(40), 1656 MASK_DECLARE_8(48), MASK_DECLARE_8(56), 1657 #endif 1658 }; 1659 EXPORT_SYMBOL_GPL(cpu_bit_bitmap); 1660 1661 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; 1662 EXPORT_SYMBOL(cpu_all_bits); 1663 1664 #ifdef CONFIG_INIT_ALL_POSSIBLE 1665 struct cpumask __cpu_possible_mask __read_mostly 1666 = {CPU_BITS_ALL}; 1667 #else 1668 struct cpumask __cpu_possible_mask __read_mostly; 1669 #endif 1670 EXPORT_SYMBOL(__cpu_possible_mask); 1671 1672 struct cpumask __cpu_online_mask __read_mostly; 1673 EXPORT_SYMBOL(__cpu_online_mask); 1674 1675 struct cpumask __cpu_present_mask __read_mostly; 1676 EXPORT_SYMBOL(__cpu_present_mask); 1677 1678 struct cpumask __cpu_active_mask __read_mostly; 1679 EXPORT_SYMBOL(__cpu_active_mask); 1680 1681 void init_cpu_present(const struct cpumask *src) 1682 { 1683 cpumask_copy(&__cpu_present_mask, src); 1684 } 1685 1686 void init_cpu_possible(const struct cpumask *src) 1687 { 1688 cpumask_copy(&__cpu_possible_mask, src); 1689 } 1690 1691 void init_cpu_online(const struct cpumask *src) 1692 { 1693 cpumask_copy(&__cpu_online_mask, src); 1694 } 1695 1696 /* 1697 * Activate the first processor. 1698 */ 1699 void __init boot_cpu_init(void) 1700 { 1701 int cpu = smp_processor_id(); 1702 1703 /* Mark the boot cpu "present", "online" etc for SMP and UP case */ 1704 set_cpu_online(cpu, true); 1705 set_cpu_active(cpu, true); 1706 set_cpu_present(cpu, true); 1707 set_cpu_possible(cpu, true); 1708 } 1709 1710 /* 1711 * Must be called _AFTER_ setting up the per_cpu areas 1712 */ 1713 void __init boot_cpu_state_init(void) 1714 { 1715 per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE; 1716 } 1717