1 /* 2 * linux/kernel/softirq.c 3 * 4 * Copyright (C) 1992 Linus Torvalds 5 * 6 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903) 7 */ 8 9 #include <linux/module.h> 10 #include <linux/kernel_stat.h> 11 #include <linux/interrupt.h> 12 #include <linux/init.h> 13 #include <linux/mm.h> 14 #include <linux/notifier.h> 15 #include <linux/percpu.h> 16 #include <linux/cpu.h> 17 #include <linux/kthread.h> 18 #include <linux/rcupdate.h> 19 #include <linux/smp.h> 20 #include <linux/tick.h> 21 22 #include <asm/irq.h> 23 /* 24 - No shared variables, all the data are CPU local. 25 - If a softirq needs serialization, let it serialize itself 26 by its own spinlocks. 27 - Even if softirq is serialized, only local cpu is marked for 28 execution. Hence, we get something sort of weak cpu binding. 29 Though it is still not clear, will it result in better locality 30 or will not. 31 32 Examples: 33 - NET RX softirq. It is multithreaded and does not require 34 any global serialization. 35 - NET TX softirq. It kicks software netdevice queues, hence 36 it is logically serialized per device, but this serialization 37 is invisible to common code. 38 - Tasklets: serialized wrt itself. 39 */ 40 41 #ifndef __ARCH_IRQ_STAT 42 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned; 43 EXPORT_SYMBOL(irq_stat); 44 #endif 45 46 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp; 47 48 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd); 49 50 /* 51 * we cannot loop indefinitely here to avoid userspace starvation, 52 * but we also don't want to introduce a worst case 1/HZ latency 53 * to the pending events, so lets the scheduler to balance 54 * the softirq load for us. 55 */ 56 static inline void wakeup_softirqd(void) 57 { 58 /* Interrupts are disabled: no need to stop preemption */ 59 struct task_struct *tsk = __get_cpu_var(ksoftirqd); 60 61 if (tsk && tsk->state != TASK_RUNNING) 62 wake_up_process(tsk); 63 } 64 65 /* 66 * This one is for softirq.c-internal use, 67 * where hardirqs are disabled legitimately: 68 */ 69 #ifdef CONFIG_TRACE_IRQFLAGS 70 static void __local_bh_disable(unsigned long ip) 71 { 72 unsigned long flags; 73 74 WARN_ON_ONCE(in_irq()); 75 76 raw_local_irq_save(flags); 77 add_preempt_count(SOFTIRQ_OFFSET); 78 /* 79 * Were softirqs turned off above: 80 */ 81 if (softirq_count() == SOFTIRQ_OFFSET) 82 trace_softirqs_off(ip); 83 raw_local_irq_restore(flags); 84 } 85 #else /* !CONFIG_TRACE_IRQFLAGS */ 86 static inline void __local_bh_disable(unsigned long ip) 87 { 88 add_preempt_count(SOFTIRQ_OFFSET); 89 barrier(); 90 } 91 #endif /* CONFIG_TRACE_IRQFLAGS */ 92 93 void local_bh_disable(void) 94 { 95 __local_bh_disable((unsigned long)__builtin_return_address(0)); 96 } 97 98 EXPORT_SYMBOL(local_bh_disable); 99 100 void __local_bh_enable(void) 101 { 102 WARN_ON_ONCE(in_irq()); 103 104 /* 105 * softirqs should never be enabled by __local_bh_enable(), 106 * it always nests inside local_bh_enable() sections: 107 */ 108 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET); 109 110 sub_preempt_count(SOFTIRQ_OFFSET); 111 } 112 EXPORT_SYMBOL_GPL(__local_bh_enable); 113 114 /* 115 * Special-case - softirqs can safely be enabled in 116 * cond_resched_softirq(), or by __do_softirq(), 117 * without processing still-pending softirqs: 118 */ 119 void _local_bh_enable(void) 120 { 121 WARN_ON_ONCE(in_irq()); 122 WARN_ON_ONCE(!irqs_disabled()); 123 124 if (softirq_count() == SOFTIRQ_OFFSET) 125 trace_softirqs_on((unsigned long)__builtin_return_address(0)); 126 sub_preempt_count(SOFTIRQ_OFFSET); 127 } 128 129 EXPORT_SYMBOL(_local_bh_enable); 130 131 void local_bh_enable(void) 132 { 133 #ifdef CONFIG_TRACE_IRQFLAGS 134 unsigned long flags; 135 136 WARN_ON_ONCE(in_irq()); 137 #endif 138 WARN_ON_ONCE(irqs_disabled()); 139 140 #ifdef CONFIG_TRACE_IRQFLAGS 141 local_irq_save(flags); 142 #endif 143 /* 144 * Are softirqs going to be turned on now: 145 */ 146 if (softirq_count() == SOFTIRQ_OFFSET) 147 trace_softirqs_on((unsigned long)__builtin_return_address(0)); 148 /* 149 * Keep preemption disabled until we are done with 150 * softirq processing: 151 */ 152 sub_preempt_count(SOFTIRQ_OFFSET - 1); 153 154 if (unlikely(!in_interrupt() && local_softirq_pending())) 155 do_softirq(); 156 157 dec_preempt_count(); 158 #ifdef CONFIG_TRACE_IRQFLAGS 159 local_irq_restore(flags); 160 #endif 161 preempt_check_resched(); 162 } 163 EXPORT_SYMBOL(local_bh_enable); 164 165 void local_bh_enable_ip(unsigned long ip) 166 { 167 #ifdef CONFIG_TRACE_IRQFLAGS 168 unsigned long flags; 169 170 WARN_ON_ONCE(in_irq()); 171 172 local_irq_save(flags); 173 #endif 174 /* 175 * Are softirqs going to be turned on now: 176 */ 177 if (softirq_count() == SOFTIRQ_OFFSET) 178 trace_softirqs_on(ip); 179 /* 180 * Keep preemption disabled until we are done with 181 * softirq processing: 182 */ 183 sub_preempt_count(SOFTIRQ_OFFSET - 1); 184 185 if (unlikely(!in_interrupt() && local_softirq_pending())) 186 do_softirq(); 187 188 dec_preempt_count(); 189 #ifdef CONFIG_TRACE_IRQFLAGS 190 local_irq_restore(flags); 191 #endif 192 preempt_check_resched(); 193 } 194 EXPORT_SYMBOL(local_bh_enable_ip); 195 196 /* 197 * We restart softirq processing MAX_SOFTIRQ_RESTART times, 198 * and we fall back to softirqd after that. 199 * 200 * This number has been established via experimentation. 201 * The two things to balance is latency against fairness - 202 * we want to handle softirqs as soon as possible, but they 203 * should not be able to lock up the box. 204 */ 205 #define MAX_SOFTIRQ_RESTART 10 206 207 asmlinkage void __do_softirq(void) 208 { 209 struct softirq_action *h; 210 __u32 pending; 211 int max_restart = MAX_SOFTIRQ_RESTART; 212 int cpu; 213 214 pending = local_softirq_pending(); 215 account_system_vtime(current); 216 217 __local_bh_disable((unsigned long)__builtin_return_address(0)); 218 trace_softirq_enter(); 219 220 cpu = smp_processor_id(); 221 restart: 222 /* Reset the pending bitmask before enabling irqs */ 223 set_softirq_pending(0); 224 225 local_irq_enable(); 226 227 h = softirq_vec; 228 229 do { 230 if (pending & 1) { 231 h->action(h); 232 rcu_bh_qsctr_inc(cpu); 233 } 234 h++; 235 pending >>= 1; 236 } while (pending); 237 238 local_irq_disable(); 239 240 pending = local_softirq_pending(); 241 if (pending && --max_restart) 242 goto restart; 243 244 if (pending) 245 wakeup_softirqd(); 246 247 trace_softirq_exit(); 248 249 account_system_vtime(current); 250 _local_bh_enable(); 251 } 252 253 #ifndef __ARCH_HAS_DO_SOFTIRQ 254 255 asmlinkage void do_softirq(void) 256 { 257 __u32 pending; 258 unsigned long flags; 259 260 if (in_interrupt()) 261 return; 262 263 local_irq_save(flags); 264 265 pending = local_softirq_pending(); 266 267 if (pending) 268 __do_softirq(); 269 270 local_irq_restore(flags); 271 } 272 273 EXPORT_SYMBOL(do_softirq); 274 275 #endif 276 277 /* 278 * Enter an interrupt context. 279 */ 280 void irq_enter(void) 281 { 282 __irq_enter(); 283 #ifdef CONFIG_NO_HZ 284 if (idle_cpu(smp_processor_id())) 285 tick_nohz_update_jiffies(); 286 #endif 287 } 288 289 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED 290 # define invoke_softirq() __do_softirq() 291 #else 292 # define invoke_softirq() do_softirq() 293 #endif 294 295 /* 296 * Exit an interrupt context. Process softirqs if needed and possible: 297 */ 298 void irq_exit(void) 299 { 300 account_system_vtime(current); 301 trace_hardirq_exit(); 302 sub_preempt_count(IRQ_EXIT_OFFSET); 303 if (!in_interrupt() && local_softirq_pending()) 304 invoke_softirq(); 305 306 #ifdef CONFIG_NO_HZ 307 /* Make sure that timer wheel updates are propagated */ 308 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched()) 309 tick_nohz_stop_sched_tick(); 310 #endif 311 preempt_enable_no_resched(); 312 } 313 314 /* 315 * This function must run with irqs disabled! 316 */ 317 inline fastcall void raise_softirq_irqoff(unsigned int nr) 318 { 319 __raise_softirq_irqoff(nr); 320 321 /* 322 * If we're in an interrupt or softirq, we're done 323 * (this also catches softirq-disabled code). We will 324 * actually run the softirq once we return from 325 * the irq or softirq. 326 * 327 * Otherwise we wake up ksoftirqd to make sure we 328 * schedule the softirq soon. 329 */ 330 if (!in_interrupt()) 331 wakeup_softirqd(); 332 } 333 334 EXPORT_SYMBOL(raise_softirq_irqoff); 335 336 void fastcall raise_softirq(unsigned int nr) 337 { 338 unsigned long flags; 339 340 local_irq_save(flags); 341 raise_softirq_irqoff(nr); 342 local_irq_restore(flags); 343 } 344 345 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data) 346 { 347 softirq_vec[nr].data = data; 348 softirq_vec[nr].action = action; 349 } 350 351 /* Tasklets */ 352 struct tasklet_head 353 { 354 struct tasklet_struct *list; 355 }; 356 357 /* Some compilers disobey section attribute on statics when not 358 initialized -- RR */ 359 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL }; 360 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL }; 361 362 void fastcall __tasklet_schedule(struct tasklet_struct *t) 363 { 364 unsigned long flags; 365 366 local_irq_save(flags); 367 t->next = __get_cpu_var(tasklet_vec).list; 368 __get_cpu_var(tasklet_vec).list = t; 369 raise_softirq_irqoff(TASKLET_SOFTIRQ); 370 local_irq_restore(flags); 371 } 372 373 EXPORT_SYMBOL(__tasklet_schedule); 374 375 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t) 376 { 377 unsigned long flags; 378 379 local_irq_save(flags); 380 t->next = __get_cpu_var(tasklet_hi_vec).list; 381 __get_cpu_var(tasklet_hi_vec).list = t; 382 raise_softirq_irqoff(HI_SOFTIRQ); 383 local_irq_restore(flags); 384 } 385 386 EXPORT_SYMBOL(__tasklet_hi_schedule); 387 388 static void tasklet_action(struct softirq_action *a) 389 { 390 struct tasklet_struct *list; 391 392 local_irq_disable(); 393 list = __get_cpu_var(tasklet_vec).list; 394 __get_cpu_var(tasklet_vec).list = NULL; 395 local_irq_enable(); 396 397 while (list) { 398 struct tasklet_struct *t = list; 399 400 list = list->next; 401 402 if (tasklet_trylock(t)) { 403 if (!atomic_read(&t->count)) { 404 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) 405 BUG(); 406 t->func(t->data); 407 tasklet_unlock(t); 408 continue; 409 } 410 tasklet_unlock(t); 411 } 412 413 local_irq_disable(); 414 t->next = __get_cpu_var(tasklet_vec).list; 415 __get_cpu_var(tasklet_vec).list = t; 416 __raise_softirq_irqoff(TASKLET_SOFTIRQ); 417 local_irq_enable(); 418 } 419 } 420 421 static void tasklet_hi_action(struct softirq_action *a) 422 { 423 struct tasklet_struct *list; 424 425 local_irq_disable(); 426 list = __get_cpu_var(tasklet_hi_vec).list; 427 __get_cpu_var(tasklet_hi_vec).list = NULL; 428 local_irq_enable(); 429 430 while (list) { 431 struct tasklet_struct *t = list; 432 433 list = list->next; 434 435 if (tasklet_trylock(t)) { 436 if (!atomic_read(&t->count)) { 437 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) 438 BUG(); 439 t->func(t->data); 440 tasklet_unlock(t); 441 continue; 442 } 443 tasklet_unlock(t); 444 } 445 446 local_irq_disable(); 447 t->next = __get_cpu_var(tasklet_hi_vec).list; 448 __get_cpu_var(tasklet_hi_vec).list = t; 449 __raise_softirq_irqoff(HI_SOFTIRQ); 450 local_irq_enable(); 451 } 452 } 453 454 455 void tasklet_init(struct tasklet_struct *t, 456 void (*func)(unsigned long), unsigned long data) 457 { 458 t->next = NULL; 459 t->state = 0; 460 atomic_set(&t->count, 0); 461 t->func = func; 462 t->data = data; 463 } 464 465 EXPORT_SYMBOL(tasklet_init); 466 467 void tasklet_kill(struct tasklet_struct *t) 468 { 469 if (in_interrupt()) 470 printk("Attempt to kill tasklet from interrupt\n"); 471 472 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { 473 do 474 yield(); 475 while (test_bit(TASKLET_STATE_SCHED, &t->state)); 476 } 477 tasklet_unlock_wait(t); 478 clear_bit(TASKLET_STATE_SCHED, &t->state); 479 } 480 481 EXPORT_SYMBOL(tasklet_kill); 482 483 void __init softirq_init(void) 484 { 485 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL); 486 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL); 487 } 488 489 static int ksoftirqd(void * __bind_cpu) 490 { 491 set_user_nice(current, 19); 492 current->flags |= PF_NOFREEZE; 493 494 set_current_state(TASK_INTERRUPTIBLE); 495 496 while (!kthread_should_stop()) { 497 preempt_disable(); 498 if (!local_softirq_pending()) { 499 preempt_enable_no_resched(); 500 schedule(); 501 preempt_disable(); 502 } 503 504 __set_current_state(TASK_RUNNING); 505 506 while (local_softirq_pending()) { 507 /* Preempt disable stops cpu going offline. 508 If already offline, we'll be on wrong CPU: 509 don't process */ 510 if (cpu_is_offline((long)__bind_cpu)) 511 goto wait_to_die; 512 do_softirq(); 513 preempt_enable_no_resched(); 514 cond_resched(); 515 preempt_disable(); 516 } 517 preempt_enable(); 518 set_current_state(TASK_INTERRUPTIBLE); 519 } 520 __set_current_state(TASK_RUNNING); 521 return 0; 522 523 wait_to_die: 524 preempt_enable(); 525 /* Wait for kthread_stop */ 526 set_current_state(TASK_INTERRUPTIBLE); 527 while (!kthread_should_stop()) { 528 schedule(); 529 set_current_state(TASK_INTERRUPTIBLE); 530 } 531 __set_current_state(TASK_RUNNING); 532 return 0; 533 } 534 535 #ifdef CONFIG_HOTPLUG_CPU 536 /* 537 * tasklet_kill_immediate is called to remove a tasklet which can already be 538 * scheduled for execution on @cpu. 539 * 540 * Unlike tasklet_kill, this function removes the tasklet 541 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state. 542 * 543 * When this function is called, @cpu must be in the CPU_DEAD state. 544 */ 545 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu) 546 { 547 struct tasklet_struct **i; 548 549 BUG_ON(cpu_online(cpu)); 550 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state)); 551 552 if (!test_bit(TASKLET_STATE_SCHED, &t->state)) 553 return; 554 555 /* CPU is dead, so no lock needed. */ 556 for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) { 557 if (*i == t) { 558 *i = t->next; 559 return; 560 } 561 } 562 BUG(); 563 } 564 565 static void takeover_tasklets(unsigned int cpu) 566 { 567 struct tasklet_struct **i; 568 569 /* CPU is dead, so no lock needed. */ 570 local_irq_disable(); 571 572 /* Find end, append list for that CPU. */ 573 for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next); 574 *i = per_cpu(tasklet_vec, cpu).list; 575 per_cpu(tasklet_vec, cpu).list = NULL; 576 raise_softirq_irqoff(TASKLET_SOFTIRQ); 577 578 for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next); 579 *i = per_cpu(tasklet_hi_vec, cpu).list; 580 per_cpu(tasklet_hi_vec, cpu).list = NULL; 581 raise_softirq_irqoff(HI_SOFTIRQ); 582 583 local_irq_enable(); 584 } 585 #endif /* CONFIG_HOTPLUG_CPU */ 586 587 static int __cpuinit cpu_callback(struct notifier_block *nfb, 588 unsigned long action, 589 void *hcpu) 590 { 591 int hotcpu = (unsigned long)hcpu; 592 struct task_struct *p; 593 594 switch (action) { 595 case CPU_UP_PREPARE: 596 case CPU_UP_PREPARE_FROZEN: 597 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu); 598 if (IS_ERR(p)) { 599 printk("ksoftirqd for %i failed\n", hotcpu); 600 return NOTIFY_BAD; 601 } 602 kthread_bind(p, hotcpu); 603 per_cpu(ksoftirqd, hotcpu) = p; 604 break; 605 case CPU_ONLINE: 606 case CPU_ONLINE_FROZEN: 607 wake_up_process(per_cpu(ksoftirqd, hotcpu)); 608 break; 609 #ifdef CONFIG_HOTPLUG_CPU 610 case CPU_UP_CANCELED: 611 case CPU_UP_CANCELED_FROZEN: 612 if (!per_cpu(ksoftirqd, hotcpu)) 613 break; 614 /* Unbind so it can run. Fall thru. */ 615 kthread_bind(per_cpu(ksoftirqd, hotcpu), 616 any_online_cpu(cpu_online_map)); 617 case CPU_DEAD: 618 case CPU_DEAD_FROZEN: 619 p = per_cpu(ksoftirqd, hotcpu); 620 per_cpu(ksoftirqd, hotcpu) = NULL; 621 kthread_stop(p); 622 takeover_tasklets(hotcpu); 623 break; 624 #endif /* CONFIG_HOTPLUG_CPU */ 625 } 626 return NOTIFY_OK; 627 } 628 629 static struct notifier_block __cpuinitdata cpu_nfb = { 630 .notifier_call = cpu_callback 631 }; 632 633 __init int spawn_ksoftirqd(void) 634 { 635 void *cpu = (void *)(long)smp_processor_id(); 636 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu); 637 638 BUG_ON(err == NOTIFY_BAD); 639 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu); 640 register_cpu_notifier(&cpu_nfb); 641 return 0; 642 } 643 644 #ifdef CONFIG_SMP 645 /* 646 * Call a function on all processors 647 */ 648 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait) 649 { 650 int ret = 0; 651 652 preempt_disable(); 653 ret = smp_call_function(func, info, retry, wait); 654 local_irq_disable(); 655 func(info); 656 local_irq_enable(); 657 preempt_enable(); 658 return ret; 659 } 660 EXPORT_SYMBOL(on_each_cpu); 661 #endif 662