1 /* 2 * linux/kernel/softirq.c 3 * 4 * Copyright (C) 1992 Linus Torvalds 5 * 6 * Distribute under GPLv2. 7 * 8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903) 9 * 10 * Remote softirq infrastructure is by Jens Axboe. 11 */ 12 13 #include <linux/module.h> 14 #include <linux/kernel_stat.h> 15 #include <linux/interrupt.h> 16 #include <linux/init.h> 17 #include <linux/mm.h> 18 #include <linux/notifier.h> 19 #include <linux/percpu.h> 20 #include <linux/cpu.h> 21 #include <linux/freezer.h> 22 #include <linux/kthread.h> 23 #include <linux/rcupdate.h> 24 #include <linux/smp.h> 25 #include <linux/tick.h> 26 27 #include <asm/irq.h> 28 /* 29 - No shared variables, all the data are CPU local. 30 - If a softirq needs serialization, let it serialize itself 31 by its own spinlocks. 32 - Even if softirq is serialized, only local cpu is marked for 33 execution. Hence, we get something sort of weak cpu binding. 34 Though it is still not clear, will it result in better locality 35 or will not. 36 37 Examples: 38 - NET RX softirq. It is multithreaded and does not require 39 any global serialization. 40 - NET TX softirq. It kicks software netdevice queues, hence 41 it is logically serialized per device, but this serialization 42 is invisible to common code. 43 - Tasklets: serialized wrt itself. 44 */ 45 46 #ifndef __ARCH_IRQ_STAT 47 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned; 48 EXPORT_SYMBOL(irq_stat); 49 #endif 50 51 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp; 52 53 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd); 54 55 /* 56 * we cannot loop indefinitely here to avoid userspace starvation, 57 * but we also don't want to introduce a worst case 1/HZ latency 58 * to the pending events, so lets the scheduler to balance 59 * the softirq load for us. 60 */ 61 static inline void wakeup_softirqd(void) 62 { 63 /* Interrupts are disabled: no need to stop preemption */ 64 struct task_struct *tsk = __get_cpu_var(ksoftirqd); 65 66 if (tsk && tsk->state != TASK_RUNNING) 67 wake_up_process(tsk); 68 } 69 70 /* 71 * This one is for softirq.c-internal use, 72 * where hardirqs are disabled legitimately: 73 */ 74 #ifdef CONFIG_TRACE_IRQFLAGS 75 static void __local_bh_disable(unsigned long ip) 76 { 77 unsigned long flags; 78 79 WARN_ON_ONCE(in_irq()); 80 81 raw_local_irq_save(flags); 82 add_preempt_count(SOFTIRQ_OFFSET); 83 /* 84 * Were softirqs turned off above: 85 */ 86 if (softirq_count() == SOFTIRQ_OFFSET) 87 trace_softirqs_off(ip); 88 raw_local_irq_restore(flags); 89 } 90 #else /* !CONFIG_TRACE_IRQFLAGS */ 91 static inline void __local_bh_disable(unsigned long ip) 92 { 93 add_preempt_count(SOFTIRQ_OFFSET); 94 barrier(); 95 } 96 #endif /* CONFIG_TRACE_IRQFLAGS */ 97 98 void local_bh_disable(void) 99 { 100 __local_bh_disable((unsigned long)__builtin_return_address(0)); 101 } 102 103 EXPORT_SYMBOL(local_bh_disable); 104 105 void __local_bh_enable(void) 106 { 107 WARN_ON_ONCE(in_irq()); 108 109 /* 110 * softirqs should never be enabled by __local_bh_enable(), 111 * it always nests inside local_bh_enable() sections: 112 */ 113 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET); 114 115 sub_preempt_count(SOFTIRQ_OFFSET); 116 } 117 EXPORT_SYMBOL_GPL(__local_bh_enable); 118 119 /* 120 * Special-case - softirqs can safely be enabled in 121 * cond_resched_softirq(), or by __do_softirq(), 122 * without processing still-pending softirqs: 123 */ 124 void _local_bh_enable(void) 125 { 126 WARN_ON_ONCE(in_irq()); 127 WARN_ON_ONCE(!irqs_disabled()); 128 129 if (softirq_count() == SOFTIRQ_OFFSET) 130 trace_softirqs_on((unsigned long)__builtin_return_address(0)); 131 sub_preempt_count(SOFTIRQ_OFFSET); 132 } 133 134 EXPORT_SYMBOL(_local_bh_enable); 135 136 static inline void _local_bh_enable_ip(unsigned long ip) 137 { 138 WARN_ON_ONCE(in_irq() || irqs_disabled()); 139 #ifdef CONFIG_TRACE_IRQFLAGS 140 local_irq_disable(); 141 #endif 142 /* 143 * Are softirqs going to be turned on now: 144 */ 145 if (softirq_count() == SOFTIRQ_OFFSET) 146 trace_softirqs_on(ip); 147 /* 148 * Keep preemption disabled until we are done with 149 * softirq processing: 150 */ 151 sub_preempt_count(SOFTIRQ_OFFSET - 1); 152 153 if (unlikely(!in_interrupt() && local_softirq_pending())) 154 do_softirq(); 155 156 dec_preempt_count(); 157 #ifdef CONFIG_TRACE_IRQFLAGS 158 local_irq_enable(); 159 #endif 160 preempt_check_resched(); 161 } 162 163 void local_bh_enable(void) 164 { 165 _local_bh_enable_ip((unsigned long)__builtin_return_address(0)); 166 } 167 EXPORT_SYMBOL(local_bh_enable); 168 169 void local_bh_enable_ip(unsigned long ip) 170 { 171 _local_bh_enable_ip(ip); 172 } 173 EXPORT_SYMBOL(local_bh_enable_ip); 174 175 /* 176 * We restart softirq processing MAX_SOFTIRQ_RESTART times, 177 * and we fall back to softirqd after that. 178 * 179 * This number has been established via experimentation. 180 * The two things to balance is latency against fairness - 181 * we want to handle softirqs as soon as possible, but they 182 * should not be able to lock up the box. 183 */ 184 #define MAX_SOFTIRQ_RESTART 10 185 186 asmlinkage void __do_softirq(void) 187 { 188 struct softirq_action *h; 189 __u32 pending; 190 int max_restart = MAX_SOFTIRQ_RESTART; 191 int cpu; 192 193 pending = local_softirq_pending(); 194 account_system_vtime(current); 195 196 __local_bh_disable((unsigned long)__builtin_return_address(0)); 197 trace_softirq_enter(); 198 199 cpu = smp_processor_id(); 200 restart: 201 /* Reset the pending bitmask before enabling irqs */ 202 set_softirq_pending(0); 203 204 local_irq_enable(); 205 206 h = softirq_vec; 207 208 do { 209 if (pending & 1) { 210 int prev_count = preempt_count(); 211 212 h->action(h); 213 214 if (unlikely(prev_count != preempt_count())) { 215 printk(KERN_ERR "huh, entered softirq %td %p" 216 "with preempt_count %08x," 217 " exited with %08x?\n", h - softirq_vec, 218 h->action, prev_count, preempt_count()); 219 preempt_count() = prev_count; 220 } 221 222 rcu_bh_qsctr_inc(cpu); 223 } 224 h++; 225 pending >>= 1; 226 } while (pending); 227 228 local_irq_disable(); 229 230 pending = local_softirq_pending(); 231 if (pending && --max_restart) 232 goto restart; 233 234 if (pending) 235 wakeup_softirqd(); 236 237 trace_softirq_exit(); 238 239 account_system_vtime(current); 240 _local_bh_enable(); 241 } 242 243 #ifndef __ARCH_HAS_DO_SOFTIRQ 244 245 asmlinkage void do_softirq(void) 246 { 247 __u32 pending; 248 unsigned long flags; 249 250 if (in_interrupt()) 251 return; 252 253 local_irq_save(flags); 254 255 pending = local_softirq_pending(); 256 257 if (pending) 258 __do_softirq(); 259 260 local_irq_restore(flags); 261 } 262 263 #endif 264 265 /* 266 * Enter an interrupt context. 267 */ 268 void irq_enter(void) 269 { 270 int cpu = smp_processor_id(); 271 272 if (idle_cpu(cpu) && !in_interrupt()) 273 tick_check_idle(cpu); 274 275 __irq_enter(); 276 } 277 278 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED 279 # define invoke_softirq() __do_softirq() 280 #else 281 # define invoke_softirq() do_softirq() 282 #endif 283 284 /* 285 * Exit an interrupt context. Process softirqs if needed and possible: 286 */ 287 void irq_exit(void) 288 { 289 account_system_vtime(current); 290 trace_hardirq_exit(); 291 sub_preempt_count(IRQ_EXIT_OFFSET); 292 if (!in_interrupt() && local_softirq_pending()) 293 invoke_softirq(); 294 295 #ifdef CONFIG_NO_HZ 296 /* Make sure that timer wheel updates are propagated */ 297 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched()) 298 tick_nohz_stop_sched_tick(0); 299 rcu_irq_exit(); 300 #endif 301 preempt_enable_no_resched(); 302 } 303 304 /* 305 * This function must run with irqs disabled! 306 */ 307 inline void raise_softirq_irqoff(unsigned int nr) 308 { 309 __raise_softirq_irqoff(nr); 310 311 /* 312 * If we're in an interrupt or softirq, we're done 313 * (this also catches softirq-disabled code). We will 314 * actually run the softirq once we return from 315 * the irq or softirq. 316 * 317 * Otherwise we wake up ksoftirqd to make sure we 318 * schedule the softirq soon. 319 */ 320 if (!in_interrupt()) 321 wakeup_softirqd(); 322 } 323 324 void raise_softirq(unsigned int nr) 325 { 326 unsigned long flags; 327 328 local_irq_save(flags); 329 raise_softirq_irqoff(nr); 330 local_irq_restore(flags); 331 } 332 333 void open_softirq(int nr, void (*action)(struct softirq_action *)) 334 { 335 softirq_vec[nr].action = action; 336 } 337 338 /* Tasklets */ 339 struct tasklet_head 340 { 341 struct tasklet_struct *head; 342 struct tasklet_struct **tail; 343 }; 344 345 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec); 346 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec); 347 348 void __tasklet_schedule(struct tasklet_struct *t) 349 { 350 unsigned long flags; 351 352 local_irq_save(flags); 353 t->next = NULL; 354 *__get_cpu_var(tasklet_vec).tail = t; 355 __get_cpu_var(tasklet_vec).tail = &(t->next); 356 raise_softirq_irqoff(TASKLET_SOFTIRQ); 357 local_irq_restore(flags); 358 } 359 360 EXPORT_SYMBOL(__tasklet_schedule); 361 362 void __tasklet_hi_schedule(struct tasklet_struct *t) 363 { 364 unsigned long flags; 365 366 local_irq_save(flags); 367 t->next = NULL; 368 *__get_cpu_var(tasklet_hi_vec).tail = t; 369 __get_cpu_var(tasklet_hi_vec).tail = &(t->next); 370 raise_softirq_irqoff(HI_SOFTIRQ); 371 local_irq_restore(flags); 372 } 373 374 EXPORT_SYMBOL(__tasklet_hi_schedule); 375 376 static void tasklet_action(struct softirq_action *a) 377 { 378 struct tasklet_struct *list; 379 380 local_irq_disable(); 381 list = __get_cpu_var(tasklet_vec).head; 382 __get_cpu_var(tasklet_vec).head = NULL; 383 __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head; 384 local_irq_enable(); 385 386 while (list) { 387 struct tasklet_struct *t = list; 388 389 list = list->next; 390 391 if (tasklet_trylock(t)) { 392 if (!atomic_read(&t->count)) { 393 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) 394 BUG(); 395 t->func(t->data); 396 tasklet_unlock(t); 397 continue; 398 } 399 tasklet_unlock(t); 400 } 401 402 local_irq_disable(); 403 t->next = NULL; 404 *__get_cpu_var(tasklet_vec).tail = t; 405 __get_cpu_var(tasklet_vec).tail = &(t->next); 406 __raise_softirq_irqoff(TASKLET_SOFTIRQ); 407 local_irq_enable(); 408 } 409 } 410 411 static void tasklet_hi_action(struct softirq_action *a) 412 { 413 struct tasklet_struct *list; 414 415 local_irq_disable(); 416 list = __get_cpu_var(tasklet_hi_vec).head; 417 __get_cpu_var(tasklet_hi_vec).head = NULL; 418 __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head; 419 local_irq_enable(); 420 421 while (list) { 422 struct tasklet_struct *t = list; 423 424 list = list->next; 425 426 if (tasklet_trylock(t)) { 427 if (!atomic_read(&t->count)) { 428 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) 429 BUG(); 430 t->func(t->data); 431 tasklet_unlock(t); 432 continue; 433 } 434 tasklet_unlock(t); 435 } 436 437 local_irq_disable(); 438 t->next = NULL; 439 *__get_cpu_var(tasklet_hi_vec).tail = t; 440 __get_cpu_var(tasklet_hi_vec).tail = &(t->next); 441 __raise_softirq_irqoff(HI_SOFTIRQ); 442 local_irq_enable(); 443 } 444 } 445 446 447 void tasklet_init(struct tasklet_struct *t, 448 void (*func)(unsigned long), unsigned long data) 449 { 450 t->next = NULL; 451 t->state = 0; 452 atomic_set(&t->count, 0); 453 t->func = func; 454 t->data = data; 455 } 456 457 EXPORT_SYMBOL(tasklet_init); 458 459 void tasklet_kill(struct tasklet_struct *t) 460 { 461 if (in_interrupt()) 462 printk("Attempt to kill tasklet from interrupt\n"); 463 464 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { 465 do 466 yield(); 467 while (test_bit(TASKLET_STATE_SCHED, &t->state)); 468 } 469 tasklet_unlock_wait(t); 470 clear_bit(TASKLET_STATE_SCHED, &t->state); 471 } 472 473 EXPORT_SYMBOL(tasklet_kill); 474 475 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list); 476 EXPORT_PER_CPU_SYMBOL(softirq_work_list); 477 478 static void __local_trigger(struct call_single_data *cp, int softirq) 479 { 480 struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]); 481 482 list_add_tail(&cp->list, head); 483 484 /* Trigger the softirq only if the list was previously empty. */ 485 if (head->next == &cp->list) 486 raise_softirq_irqoff(softirq); 487 } 488 489 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS 490 static void remote_softirq_receive(void *data) 491 { 492 struct call_single_data *cp = data; 493 unsigned long flags; 494 int softirq; 495 496 softirq = cp->priv; 497 498 local_irq_save(flags); 499 __local_trigger(cp, softirq); 500 local_irq_restore(flags); 501 } 502 503 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq) 504 { 505 if (cpu_online(cpu)) { 506 cp->func = remote_softirq_receive; 507 cp->info = cp; 508 cp->flags = 0; 509 cp->priv = softirq; 510 511 __smp_call_function_single(cpu, cp); 512 return 0; 513 } 514 return 1; 515 } 516 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */ 517 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq) 518 { 519 return 1; 520 } 521 #endif 522 523 /** 524 * __send_remote_softirq - try to schedule softirq work on a remote cpu 525 * @cp: private SMP call function data area 526 * @cpu: the remote cpu 527 * @this_cpu: the currently executing cpu 528 * @softirq: the softirq for the work 529 * 530 * Attempt to schedule softirq work on a remote cpu. If this cannot be 531 * done, the work is instead queued up on the local cpu. 532 * 533 * Interrupts must be disabled. 534 */ 535 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq) 536 { 537 if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq)) 538 __local_trigger(cp, softirq); 539 } 540 EXPORT_SYMBOL(__send_remote_softirq); 541 542 /** 543 * send_remote_softirq - try to schedule softirq work on a remote cpu 544 * @cp: private SMP call function data area 545 * @cpu: the remote cpu 546 * @softirq: the softirq for the work 547 * 548 * Like __send_remote_softirq except that disabling interrupts and 549 * computing the current cpu is done for the caller. 550 */ 551 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq) 552 { 553 unsigned long flags; 554 int this_cpu; 555 556 local_irq_save(flags); 557 this_cpu = smp_processor_id(); 558 __send_remote_softirq(cp, cpu, this_cpu, softirq); 559 local_irq_restore(flags); 560 } 561 EXPORT_SYMBOL(send_remote_softirq); 562 563 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self, 564 unsigned long action, void *hcpu) 565 { 566 /* 567 * If a CPU goes away, splice its entries to the current CPU 568 * and trigger a run of the softirq 569 */ 570 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { 571 int cpu = (unsigned long) hcpu; 572 int i; 573 574 local_irq_disable(); 575 for (i = 0; i < NR_SOFTIRQS; i++) { 576 struct list_head *head = &per_cpu(softirq_work_list[i], cpu); 577 struct list_head *local_head; 578 579 if (list_empty(head)) 580 continue; 581 582 local_head = &__get_cpu_var(softirq_work_list[i]); 583 list_splice_init(head, local_head); 584 raise_softirq_irqoff(i); 585 } 586 local_irq_enable(); 587 } 588 589 return NOTIFY_OK; 590 } 591 592 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = { 593 .notifier_call = remote_softirq_cpu_notify, 594 }; 595 596 void __init softirq_init(void) 597 { 598 int cpu; 599 600 for_each_possible_cpu(cpu) { 601 int i; 602 603 per_cpu(tasklet_vec, cpu).tail = 604 &per_cpu(tasklet_vec, cpu).head; 605 per_cpu(tasklet_hi_vec, cpu).tail = 606 &per_cpu(tasklet_hi_vec, cpu).head; 607 for (i = 0; i < NR_SOFTIRQS; i++) 608 INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu)); 609 } 610 611 register_hotcpu_notifier(&remote_softirq_cpu_notifier); 612 613 open_softirq(TASKLET_SOFTIRQ, tasklet_action); 614 open_softirq(HI_SOFTIRQ, tasklet_hi_action); 615 } 616 617 static int ksoftirqd(void * __bind_cpu) 618 { 619 set_current_state(TASK_INTERRUPTIBLE); 620 621 while (!kthread_should_stop()) { 622 preempt_disable(); 623 if (!local_softirq_pending()) { 624 preempt_enable_no_resched(); 625 schedule(); 626 preempt_disable(); 627 } 628 629 __set_current_state(TASK_RUNNING); 630 631 while (local_softirq_pending()) { 632 /* Preempt disable stops cpu going offline. 633 If already offline, we'll be on wrong CPU: 634 don't process */ 635 if (cpu_is_offline((long)__bind_cpu)) 636 goto wait_to_die; 637 do_softirq(); 638 preempt_enable_no_resched(); 639 cond_resched(); 640 preempt_disable(); 641 } 642 preempt_enable(); 643 set_current_state(TASK_INTERRUPTIBLE); 644 } 645 __set_current_state(TASK_RUNNING); 646 return 0; 647 648 wait_to_die: 649 preempt_enable(); 650 /* Wait for kthread_stop */ 651 set_current_state(TASK_INTERRUPTIBLE); 652 while (!kthread_should_stop()) { 653 schedule(); 654 set_current_state(TASK_INTERRUPTIBLE); 655 } 656 __set_current_state(TASK_RUNNING); 657 return 0; 658 } 659 660 #ifdef CONFIG_HOTPLUG_CPU 661 /* 662 * tasklet_kill_immediate is called to remove a tasklet which can already be 663 * scheduled for execution on @cpu. 664 * 665 * Unlike tasklet_kill, this function removes the tasklet 666 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state. 667 * 668 * When this function is called, @cpu must be in the CPU_DEAD state. 669 */ 670 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu) 671 { 672 struct tasklet_struct **i; 673 674 BUG_ON(cpu_online(cpu)); 675 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state)); 676 677 if (!test_bit(TASKLET_STATE_SCHED, &t->state)) 678 return; 679 680 /* CPU is dead, so no lock needed. */ 681 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) { 682 if (*i == t) { 683 *i = t->next; 684 /* If this was the tail element, move the tail ptr */ 685 if (*i == NULL) 686 per_cpu(tasklet_vec, cpu).tail = i; 687 return; 688 } 689 } 690 BUG(); 691 } 692 693 static void takeover_tasklets(unsigned int cpu) 694 { 695 /* CPU is dead, so no lock needed. */ 696 local_irq_disable(); 697 698 /* Find end, append list for that CPU. */ 699 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) { 700 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head; 701 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail; 702 per_cpu(tasklet_vec, cpu).head = NULL; 703 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head; 704 } 705 raise_softirq_irqoff(TASKLET_SOFTIRQ); 706 707 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) { 708 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head; 709 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail; 710 per_cpu(tasklet_hi_vec, cpu).head = NULL; 711 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head; 712 } 713 raise_softirq_irqoff(HI_SOFTIRQ); 714 715 local_irq_enable(); 716 } 717 #endif /* CONFIG_HOTPLUG_CPU */ 718 719 static int __cpuinit cpu_callback(struct notifier_block *nfb, 720 unsigned long action, 721 void *hcpu) 722 { 723 int hotcpu = (unsigned long)hcpu; 724 struct task_struct *p; 725 726 switch (action) { 727 case CPU_UP_PREPARE: 728 case CPU_UP_PREPARE_FROZEN: 729 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu); 730 if (IS_ERR(p)) { 731 printk("ksoftirqd for %i failed\n", hotcpu); 732 return NOTIFY_BAD; 733 } 734 kthread_bind(p, hotcpu); 735 per_cpu(ksoftirqd, hotcpu) = p; 736 break; 737 case CPU_ONLINE: 738 case CPU_ONLINE_FROZEN: 739 wake_up_process(per_cpu(ksoftirqd, hotcpu)); 740 break; 741 #ifdef CONFIG_HOTPLUG_CPU 742 case CPU_UP_CANCELED: 743 case CPU_UP_CANCELED_FROZEN: 744 if (!per_cpu(ksoftirqd, hotcpu)) 745 break; 746 /* Unbind so it can run. Fall thru. */ 747 kthread_bind(per_cpu(ksoftirqd, hotcpu), 748 any_online_cpu(cpu_online_map)); 749 case CPU_DEAD: 750 case CPU_DEAD_FROZEN: { 751 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; 752 753 p = per_cpu(ksoftirqd, hotcpu); 754 per_cpu(ksoftirqd, hotcpu) = NULL; 755 sched_setscheduler_nocheck(p, SCHED_FIFO, ¶m); 756 kthread_stop(p); 757 takeover_tasklets(hotcpu); 758 break; 759 } 760 #endif /* CONFIG_HOTPLUG_CPU */ 761 } 762 return NOTIFY_OK; 763 } 764 765 static struct notifier_block __cpuinitdata cpu_nfb = { 766 .notifier_call = cpu_callback 767 }; 768 769 static __init int spawn_ksoftirqd(void) 770 { 771 void *cpu = (void *)(long)smp_processor_id(); 772 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu); 773 774 BUG_ON(err == NOTIFY_BAD); 775 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu); 776 register_cpu_notifier(&cpu_nfb); 777 return 0; 778 } 779 early_initcall(spawn_ksoftirqd); 780 781 #ifdef CONFIG_SMP 782 /* 783 * Call a function on all processors 784 */ 785 int on_each_cpu(void (*func) (void *info), void *info, int wait) 786 { 787 int ret = 0; 788 789 preempt_disable(); 790 ret = smp_call_function(func, info, wait); 791 local_irq_disable(); 792 func(info); 793 local_irq_enable(); 794 preempt_enable(); 795 return ret; 796 } 797 EXPORT_SYMBOL(on_each_cpu); 798 #endif 799