1 /* 2 * linux/kernel/softirq.c 3 * 4 * Copyright (C) 1992 Linus Torvalds 5 * 6 * Distribute under GPLv2. 7 * 8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903) 9 * 10 * Remote softirq infrastructure is by Jens Axboe. 11 */ 12 13 #include <linux/export.h> 14 #include <linux/kernel_stat.h> 15 #include <linux/interrupt.h> 16 #include <linux/init.h> 17 #include <linux/mm.h> 18 #include <linux/notifier.h> 19 #include <linux/percpu.h> 20 #include <linux/cpu.h> 21 #include <linux/freezer.h> 22 #include <linux/kthread.h> 23 #include <linux/rcupdate.h> 24 #include <linux/ftrace.h> 25 #include <linux/smp.h> 26 #include <linux/smpboot.h> 27 #include <linux/tick.h> 28 29 #define CREATE_TRACE_POINTS 30 #include <trace/events/irq.h> 31 32 #include <asm/irq.h> 33 /* 34 - No shared variables, all the data are CPU local. 35 - If a softirq needs serialization, let it serialize itself 36 by its own spinlocks. 37 - Even if softirq is serialized, only local cpu is marked for 38 execution. Hence, we get something sort of weak cpu binding. 39 Though it is still not clear, will it result in better locality 40 or will not. 41 42 Examples: 43 - NET RX softirq. It is multithreaded and does not require 44 any global serialization. 45 - NET TX softirq. It kicks software netdevice queues, hence 46 it is logically serialized per device, but this serialization 47 is invisible to common code. 48 - Tasklets: serialized wrt itself. 49 */ 50 51 #ifndef __ARCH_IRQ_STAT 52 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned; 53 EXPORT_SYMBOL(irq_stat); 54 #endif 55 56 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp; 57 58 DEFINE_PER_CPU(struct task_struct *, ksoftirqd); 59 60 char *softirq_to_name[NR_SOFTIRQS] = { 61 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL", 62 "TASKLET", "SCHED", "HRTIMER", "RCU" 63 }; 64 65 /* 66 * we cannot loop indefinitely here to avoid userspace starvation, 67 * but we also don't want to introduce a worst case 1/HZ latency 68 * to the pending events, so lets the scheduler to balance 69 * the softirq load for us. 70 */ 71 static void wakeup_softirqd(void) 72 { 73 /* Interrupts are disabled: no need to stop preemption */ 74 struct task_struct *tsk = __this_cpu_read(ksoftirqd); 75 76 if (tsk && tsk->state != TASK_RUNNING) 77 wake_up_process(tsk); 78 } 79 80 /* 81 * preempt_count and SOFTIRQ_OFFSET usage: 82 * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving 83 * softirq processing. 84 * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET) 85 * on local_bh_disable or local_bh_enable. 86 * This lets us distinguish between whether we are currently processing 87 * softirq and whether we just have bh disabled. 88 */ 89 90 /* 91 * This one is for softirq.c-internal use, 92 * where hardirqs are disabled legitimately: 93 */ 94 #ifdef CONFIG_TRACE_IRQFLAGS 95 static void __local_bh_disable(unsigned long ip, unsigned int cnt) 96 { 97 unsigned long flags; 98 99 WARN_ON_ONCE(in_irq()); 100 101 raw_local_irq_save(flags); 102 /* 103 * The preempt tracer hooks into add_preempt_count and will break 104 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET 105 * is set and before current->softirq_enabled is cleared. 106 * We must manually increment preempt_count here and manually 107 * call the trace_preempt_off later. 108 */ 109 preempt_count() += cnt; 110 /* 111 * Were softirqs turned off above: 112 */ 113 if (softirq_count() == cnt) 114 trace_softirqs_off(ip); 115 raw_local_irq_restore(flags); 116 117 if (preempt_count() == cnt) 118 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); 119 } 120 #else /* !CONFIG_TRACE_IRQFLAGS */ 121 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt) 122 { 123 add_preempt_count(cnt); 124 barrier(); 125 } 126 #endif /* CONFIG_TRACE_IRQFLAGS */ 127 128 void local_bh_disable(void) 129 { 130 __local_bh_disable((unsigned long)__builtin_return_address(0), 131 SOFTIRQ_DISABLE_OFFSET); 132 } 133 134 EXPORT_SYMBOL(local_bh_disable); 135 136 static void __local_bh_enable(unsigned int cnt) 137 { 138 WARN_ON_ONCE(in_irq()); 139 WARN_ON_ONCE(!irqs_disabled()); 140 141 if (softirq_count() == cnt) 142 trace_softirqs_on((unsigned long)__builtin_return_address(0)); 143 sub_preempt_count(cnt); 144 } 145 146 /* 147 * Special-case - softirqs can safely be enabled in 148 * cond_resched_softirq(), or by __do_softirq(), 149 * without processing still-pending softirqs: 150 */ 151 void _local_bh_enable(void) 152 { 153 __local_bh_enable(SOFTIRQ_DISABLE_OFFSET); 154 } 155 156 EXPORT_SYMBOL(_local_bh_enable); 157 158 static inline void _local_bh_enable_ip(unsigned long ip) 159 { 160 WARN_ON_ONCE(in_irq() || irqs_disabled()); 161 #ifdef CONFIG_TRACE_IRQFLAGS 162 local_irq_disable(); 163 #endif 164 /* 165 * Are softirqs going to be turned on now: 166 */ 167 if (softirq_count() == SOFTIRQ_DISABLE_OFFSET) 168 trace_softirqs_on(ip); 169 /* 170 * Keep preemption disabled until we are done with 171 * softirq processing: 172 */ 173 sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1); 174 175 if (unlikely(!in_interrupt() && local_softirq_pending())) 176 do_softirq(); 177 178 dec_preempt_count(); 179 #ifdef CONFIG_TRACE_IRQFLAGS 180 local_irq_enable(); 181 #endif 182 preempt_check_resched(); 183 } 184 185 void local_bh_enable(void) 186 { 187 _local_bh_enable_ip((unsigned long)__builtin_return_address(0)); 188 } 189 EXPORT_SYMBOL(local_bh_enable); 190 191 void local_bh_enable_ip(unsigned long ip) 192 { 193 _local_bh_enable_ip(ip); 194 } 195 EXPORT_SYMBOL(local_bh_enable_ip); 196 197 /* 198 * We restart softirq processing for at most 2 ms, 199 * and if need_resched() is not set. 200 * 201 * These limits have been established via experimentation. 202 * The two things to balance is latency against fairness - 203 * we want to handle softirqs as soon as possible, but they 204 * should not be able to lock up the box. 205 */ 206 #define MAX_SOFTIRQ_TIME msecs_to_jiffies(2) 207 208 asmlinkage void __do_softirq(void) 209 { 210 struct softirq_action *h; 211 __u32 pending; 212 unsigned long end = jiffies + MAX_SOFTIRQ_TIME; 213 int cpu; 214 unsigned long old_flags = current->flags; 215 216 /* 217 * Mask out PF_MEMALLOC s current task context is borrowed for the 218 * softirq. A softirq handled such as network RX might set PF_MEMALLOC 219 * again if the socket is related to swap 220 */ 221 current->flags &= ~PF_MEMALLOC; 222 223 pending = local_softirq_pending(); 224 account_irq_enter_time(current); 225 226 __local_bh_disable((unsigned long)__builtin_return_address(0), 227 SOFTIRQ_OFFSET); 228 lockdep_softirq_enter(); 229 230 cpu = smp_processor_id(); 231 restart: 232 /* Reset the pending bitmask before enabling irqs */ 233 set_softirq_pending(0); 234 235 local_irq_enable(); 236 237 h = softirq_vec; 238 239 do { 240 if (pending & 1) { 241 unsigned int vec_nr = h - softirq_vec; 242 int prev_count = preempt_count(); 243 244 kstat_incr_softirqs_this_cpu(vec_nr); 245 246 trace_softirq_entry(vec_nr); 247 h->action(h); 248 trace_softirq_exit(vec_nr); 249 if (unlikely(prev_count != preempt_count())) { 250 printk(KERN_ERR "huh, entered softirq %u %s %p" 251 "with preempt_count %08x," 252 " exited with %08x?\n", vec_nr, 253 softirq_to_name[vec_nr], h->action, 254 prev_count, preempt_count()); 255 preempt_count() = prev_count; 256 } 257 258 rcu_bh_qs(cpu); 259 } 260 h++; 261 pending >>= 1; 262 } while (pending); 263 264 local_irq_disable(); 265 266 pending = local_softirq_pending(); 267 if (pending) { 268 if (time_before(jiffies, end) && !need_resched()) 269 goto restart; 270 271 wakeup_softirqd(); 272 } 273 274 lockdep_softirq_exit(); 275 276 account_irq_exit_time(current); 277 __local_bh_enable(SOFTIRQ_OFFSET); 278 tsk_restore_flags(current, old_flags, PF_MEMALLOC); 279 } 280 281 #ifndef __ARCH_HAS_DO_SOFTIRQ 282 283 asmlinkage void do_softirq(void) 284 { 285 __u32 pending; 286 unsigned long flags; 287 288 if (in_interrupt()) 289 return; 290 291 local_irq_save(flags); 292 293 pending = local_softirq_pending(); 294 295 if (pending) 296 __do_softirq(); 297 298 local_irq_restore(flags); 299 } 300 301 #endif 302 303 /* 304 * Enter an interrupt context. 305 */ 306 void irq_enter(void) 307 { 308 int cpu = smp_processor_id(); 309 310 rcu_irq_enter(); 311 if (is_idle_task(current) && !in_interrupt()) { 312 /* 313 * Prevent raise_softirq from needlessly waking up ksoftirqd 314 * here, as softirq will be serviced on return from interrupt. 315 */ 316 local_bh_disable(); 317 tick_check_idle(cpu); 318 _local_bh_enable(); 319 } 320 321 __irq_enter(); 322 } 323 324 static inline void invoke_softirq(void) 325 { 326 if (!force_irqthreads) 327 __do_softirq(); 328 else 329 wakeup_softirqd(); 330 } 331 332 static inline void tick_irq_exit(void) 333 { 334 #ifdef CONFIG_NO_HZ_COMMON 335 int cpu = smp_processor_id(); 336 337 /* Make sure that timer wheel updates are propagated */ 338 if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) { 339 if (!in_interrupt()) 340 tick_nohz_irq_exit(); 341 } 342 #endif 343 } 344 345 /* 346 * Exit an interrupt context. Process softirqs if needed and possible: 347 */ 348 void irq_exit(void) 349 { 350 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED 351 local_irq_disable(); 352 #else 353 WARN_ON_ONCE(!irqs_disabled()); 354 #endif 355 356 account_irq_exit_time(current); 357 trace_hardirq_exit(); 358 sub_preempt_count(HARDIRQ_OFFSET); 359 if (!in_interrupt() && local_softirq_pending()) 360 invoke_softirq(); 361 362 tick_irq_exit(); 363 rcu_irq_exit(); 364 } 365 366 /* 367 * This function must run with irqs disabled! 368 */ 369 inline void raise_softirq_irqoff(unsigned int nr) 370 { 371 __raise_softirq_irqoff(nr); 372 373 /* 374 * If we're in an interrupt or softirq, we're done 375 * (this also catches softirq-disabled code). We will 376 * actually run the softirq once we return from 377 * the irq or softirq. 378 * 379 * Otherwise we wake up ksoftirqd to make sure we 380 * schedule the softirq soon. 381 */ 382 if (!in_interrupt()) 383 wakeup_softirqd(); 384 } 385 386 void raise_softirq(unsigned int nr) 387 { 388 unsigned long flags; 389 390 local_irq_save(flags); 391 raise_softirq_irqoff(nr); 392 local_irq_restore(flags); 393 } 394 395 void __raise_softirq_irqoff(unsigned int nr) 396 { 397 trace_softirq_raise(nr); 398 or_softirq_pending(1UL << nr); 399 } 400 401 void open_softirq(int nr, void (*action)(struct softirq_action *)) 402 { 403 softirq_vec[nr].action = action; 404 } 405 406 /* 407 * Tasklets 408 */ 409 struct tasklet_head 410 { 411 struct tasklet_struct *head; 412 struct tasklet_struct **tail; 413 }; 414 415 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec); 416 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec); 417 418 void __tasklet_schedule(struct tasklet_struct *t) 419 { 420 unsigned long flags; 421 422 local_irq_save(flags); 423 t->next = NULL; 424 *__this_cpu_read(tasklet_vec.tail) = t; 425 __this_cpu_write(tasklet_vec.tail, &(t->next)); 426 raise_softirq_irqoff(TASKLET_SOFTIRQ); 427 local_irq_restore(flags); 428 } 429 430 EXPORT_SYMBOL(__tasklet_schedule); 431 432 void __tasklet_hi_schedule(struct tasklet_struct *t) 433 { 434 unsigned long flags; 435 436 local_irq_save(flags); 437 t->next = NULL; 438 *__this_cpu_read(tasklet_hi_vec.tail) = t; 439 __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); 440 raise_softirq_irqoff(HI_SOFTIRQ); 441 local_irq_restore(flags); 442 } 443 444 EXPORT_SYMBOL(__tasklet_hi_schedule); 445 446 void __tasklet_hi_schedule_first(struct tasklet_struct *t) 447 { 448 BUG_ON(!irqs_disabled()); 449 450 t->next = __this_cpu_read(tasklet_hi_vec.head); 451 __this_cpu_write(tasklet_hi_vec.head, t); 452 __raise_softirq_irqoff(HI_SOFTIRQ); 453 } 454 455 EXPORT_SYMBOL(__tasklet_hi_schedule_first); 456 457 static void tasklet_action(struct softirq_action *a) 458 { 459 struct tasklet_struct *list; 460 461 local_irq_disable(); 462 list = __this_cpu_read(tasklet_vec.head); 463 __this_cpu_write(tasklet_vec.head, NULL); 464 __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head); 465 local_irq_enable(); 466 467 while (list) { 468 struct tasklet_struct *t = list; 469 470 list = list->next; 471 472 if (tasklet_trylock(t)) { 473 if (!atomic_read(&t->count)) { 474 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) 475 BUG(); 476 t->func(t->data); 477 tasklet_unlock(t); 478 continue; 479 } 480 tasklet_unlock(t); 481 } 482 483 local_irq_disable(); 484 t->next = NULL; 485 *__this_cpu_read(tasklet_vec.tail) = t; 486 __this_cpu_write(tasklet_vec.tail, &(t->next)); 487 __raise_softirq_irqoff(TASKLET_SOFTIRQ); 488 local_irq_enable(); 489 } 490 } 491 492 static void tasklet_hi_action(struct softirq_action *a) 493 { 494 struct tasklet_struct *list; 495 496 local_irq_disable(); 497 list = __this_cpu_read(tasklet_hi_vec.head); 498 __this_cpu_write(tasklet_hi_vec.head, NULL); 499 __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head); 500 local_irq_enable(); 501 502 while (list) { 503 struct tasklet_struct *t = list; 504 505 list = list->next; 506 507 if (tasklet_trylock(t)) { 508 if (!atomic_read(&t->count)) { 509 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) 510 BUG(); 511 t->func(t->data); 512 tasklet_unlock(t); 513 continue; 514 } 515 tasklet_unlock(t); 516 } 517 518 local_irq_disable(); 519 t->next = NULL; 520 *__this_cpu_read(tasklet_hi_vec.tail) = t; 521 __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); 522 __raise_softirq_irqoff(HI_SOFTIRQ); 523 local_irq_enable(); 524 } 525 } 526 527 528 void tasklet_init(struct tasklet_struct *t, 529 void (*func)(unsigned long), unsigned long data) 530 { 531 t->next = NULL; 532 t->state = 0; 533 atomic_set(&t->count, 0); 534 t->func = func; 535 t->data = data; 536 } 537 538 EXPORT_SYMBOL(tasklet_init); 539 540 void tasklet_kill(struct tasklet_struct *t) 541 { 542 if (in_interrupt()) 543 printk("Attempt to kill tasklet from interrupt\n"); 544 545 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { 546 do { 547 yield(); 548 } while (test_bit(TASKLET_STATE_SCHED, &t->state)); 549 } 550 tasklet_unlock_wait(t); 551 clear_bit(TASKLET_STATE_SCHED, &t->state); 552 } 553 554 EXPORT_SYMBOL(tasklet_kill); 555 556 /* 557 * tasklet_hrtimer 558 */ 559 560 /* 561 * The trampoline is called when the hrtimer expires. It schedules a tasklet 562 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended 563 * hrtimer callback, but from softirq context. 564 */ 565 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer) 566 { 567 struct tasklet_hrtimer *ttimer = 568 container_of(timer, struct tasklet_hrtimer, timer); 569 570 tasklet_hi_schedule(&ttimer->tasklet); 571 return HRTIMER_NORESTART; 572 } 573 574 /* 575 * Helper function which calls the hrtimer callback from 576 * tasklet/softirq context 577 */ 578 static void __tasklet_hrtimer_trampoline(unsigned long data) 579 { 580 struct tasklet_hrtimer *ttimer = (void *)data; 581 enum hrtimer_restart restart; 582 583 restart = ttimer->function(&ttimer->timer); 584 if (restart != HRTIMER_NORESTART) 585 hrtimer_restart(&ttimer->timer); 586 } 587 588 /** 589 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks 590 * @ttimer: tasklet_hrtimer which is initialized 591 * @function: hrtimer callback function which gets called from softirq context 592 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME) 593 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL) 594 */ 595 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer, 596 enum hrtimer_restart (*function)(struct hrtimer *), 597 clockid_t which_clock, enum hrtimer_mode mode) 598 { 599 hrtimer_init(&ttimer->timer, which_clock, mode); 600 ttimer->timer.function = __hrtimer_tasklet_trampoline; 601 tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline, 602 (unsigned long)ttimer); 603 ttimer->function = function; 604 } 605 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init); 606 607 /* 608 * Remote softirq bits 609 */ 610 611 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list); 612 EXPORT_PER_CPU_SYMBOL(softirq_work_list); 613 614 static void __local_trigger(struct call_single_data *cp, int softirq) 615 { 616 struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]); 617 618 list_add_tail(&cp->list, head); 619 620 /* Trigger the softirq only if the list was previously empty. */ 621 if (head->next == &cp->list) 622 raise_softirq_irqoff(softirq); 623 } 624 625 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS 626 static void remote_softirq_receive(void *data) 627 { 628 struct call_single_data *cp = data; 629 unsigned long flags; 630 int softirq; 631 632 softirq = *(int *)cp->info; 633 local_irq_save(flags); 634 __local_trigger(cp, softirq); 635 local_irq_restore(flags); 636 } 637 638 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq) 639 { 640 if (cpu_online(cpu)) { 641 cp->func = remote_softirq_receive; 642 cp->info = &softirq; 643 cp->flags = 0; 644 645 __smp_call_function_single(cpu, cp, 0); 646 return 0; 647 } 648 return 1; 649 } 650 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */ 651 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq) 652 { 653 return 1; 654 } 655 #endif 656 657 /** 658 * __send_remote_softirq - try to schedule softirq work on a remote cpu 659 * @cp: private SMP call function data area 660 * @cpu: the remote cpu 661 * @this_cpu: the currently executing cpu 662 * @softirq: the softirq for the work 663 * 664 * Attempt to schedule softirq work on a remote cpu. If this cannot be 665 * done, the work is instead queued up on the local cpu. 666 * 667 * Interrupts must be disabled. 668 */ 669 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq) 670 { 671 if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq)) 672 __local_trigger(cp, softirq); 673 } 674 EXPORT_SYMBOL(__send_remote_softirq); 675 676 /** 677 * send_remote_softirq - try to schedule softirq work on a remote cpu 678 * @cp: private SMP call function data area 679 * @cpu: the remote cpu 680 * @softirq: the softirq for the work 681 * 682 * Like __send_remote_softirq except that disabling interrupts and 683 * computing the current cpu is done for the caller. 684 */ 685 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq) 686 { 687 unsigned long flags; 688 int this_cpu; 689 690 local_irq_save(flags); 691 this_cpu = smp_processor_id(); 692 __send_remote_softirq(cp, cpu, this_cpu, softirq); 693 local_irq_restore(flags); 694 } 695 EXPORT_SYMBOL(send_remote_softirq); 696 697 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self, 698 unsigned long action, void *hcpu) 699 { 700 /* 701 * If a CPU goes away, splice its entries to the current CPU 702 * and trigger a run of the softirq 703 */ 704 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { 705 int cpu = (unsigned long) hcpu; 706 int i; 707 708 local_irq_disable(); 709 for (i = 0; i < NR_SOFTIRQS; i++) { 710 struct list_head *head = &per_cpu(softirq_work_list[i], cpu); 711 struct list_head *local_head; 712 713 if (list_empty(head)) 714 continue; 715 716 local_head = &__get_cpu_var(softirq_work_list[i]); 717 list_splice_init(head, local_head); 718 raise_softirq_irqoff(i); 719 } 720 local_irq_enable(); 721 } 722 723 return NOTIFY_OK; 724 } 725 726 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = { 727 .notifier_call = remote_softirq_cpu_notify, 728 }; 729 730 void __init softirq_init(void) 731 { 732 int cpu; 733 734 for_each_possible_cpu(cpu) { 735 int i; 736 737 per_cpu(tasklet_vec, cpu).tail = 738 &per_cpu(tasklet_vec, cpu).head; 739 per_cpu(tasklet_hi_vec, cpu).tail = 740 &per_cpu(tasklet_hi_vec, cpu).head; 741 for (i = 0; i < NR_SOFTIRQS; i++) 742 INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu)); 743 } 744 745 register_hotcpu_notifier(&remote_softirq_cpu_notifier); 746 747 open_softirq(TASKLET_SOFTIRQ, tasklet_action); 748 open_softirq(HI_SOFTIRQ, tasklet_hi_action); 749 } 750 751 static int ksoftirqd_should_run(unsigned int cpu) 752 { 753 return local_softirq_pending(); 754 } 755 756 static void run_ksoftirqd(unsigned int cpu) 757 { 758 local_irq_disable(); 759 if (local_softirq_pending()) { 760 __do_softirq(); 761 rcu_note_context_switch(cpu); 762 local_irq_enable(); 763 cond_resched(); 764 return; 765 } 766 local_irq_enable(); 767 } 768 769 #ifdef CONFIG_HOTPLUG_CPU 770 /* 771 * tasklet_kill_immediate is called to remove a tasklet which can already be 772 * scheduled for execution on @cpu. 773 * 774 * Unlike tasklet_kill, this function removes the tasklet 775 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state. 776 * 777 * When this function is called, @cpu must be in the CPU_DEAD state. 778 */ 779 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu) 780 { 781 struct tasklet_struct **i; 782 783 BUG_ON(cpu_online(cpu)); 784 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state)); 785 786 if (!test_bit(TASKLET_STATE_SCHED, &t->state)) 787 return; 788 789 /* CPU is dead, so no lock needed. */ 790 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) { 791 if (*i == t) { 792 *i = t->next; 793 /* If this was the tail element, move the tail ptr */ 794 if (*i == NULL) 795 per_cpu(tasklet_vec, cpu).tail = i; 796 return; 797 } 798 } 799 BUG(); 800 } 801 802 static void takeover_tasklets(unsigned int cpu) 803 { 804 /* CPU is dead, so no lock needed. */ 805 local_irq_disable(); 806 807 /* Find end, append list for that CPU. */ 808 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) { 809 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head; 810 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail); 811 per_cpu(tasklet_vec, cpu).head = NULL; 812 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head; 813 } 814 raise_softirq_irqoff(TASKLET_SOFTIRQ); 815 816 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) { 817 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head; 818 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail); 819 per_cpu(tasklet_hi_vec, cpu).head = NULL; 820 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head; 821 } 822 raise_softirq_irqoff(HI_SOFTIRQ); 823 824 local_irq_enable(); 825 } 826 #endif /* CONFIG_HOTPLUG_CPU */ 827 828 static int __cpuinit cpu_callback(struct notifier_block *nfb, 829 unsigned long action, 830 void *hcpu) 831 { 832 switch (action) { 833 #ifdef CONFIG_HOTPLUG_CPU 834 case CPU_DEAD: 835 case CPU_DEAD_FROZEN: 836 takeover_tasklets((unsigned long)hcpu); 837 break; 838 #endif /* CONFIG_HOTPLUG_CPU */ 839 } 840 return NOTIFY_OK; 841 } 842 843 static struct notifier_block __cpuinitdata cpu_nfb = { 844 .notifier_call = cpu_callback 845 }; 846 847 static struct smp_hotplug_thread softirq_threads = { 848 .store = &ksoftirqd, 849 .thread_should_run = ksoftirqd_should_run, 850 .thread_fn = run_ksoftirqd, 851 .thread_comm = "ksoftirqd/%u", 852 }; 853 854 static __init int spawn_ksoftirqd(void) 855 { 856 register_cpu_notifier(&cpu_nfb); 857 858 BUG_ON(smpboot_register_percpu_thread(&softirq_threads)); 859 860 return 0; 861 } 862 early_initcall(spawn_ksoftirqd); 863 864 /* 865 * [ These __weak aliases are kept in a separate compilation unit, so that 866 * GCC does not inline them incorrectly. ] 867 */ 868 869 int __init __weak early_irq_init(void) 870 { 871 return 0; 872 } 873 874 #ifdef CONFIG_GENERIC_HARDIRQS 875 int __init __weak arch_probe_nr_irqs(void) 876 { 877 return NR_IRQS_LEGACY; 878 } 879 880 int __init __weak arch_early_irq_init(void) 881 { 882 return 0; 883 } 884 #endif 885