xref: /openbmc/linux/kernel/softirq.c (revision e868d61272caa648214046a096e5a6bfc068dc8c)
1 /*
2  *	linux/kernel/softirq.c
3  *
4  *	Copyright (C) 1992 Linus Torvalds
5  *
6  * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
7  */
8 
9 #include <linux/module.h>
10 #include <linux/kernel_stat.h>
11 #include <linux/interrupt.h>
12 #include <linux/init.h>
13 #include <linux/mm.h>
14 #include <linux/notifier.h>
15 #include <linux/percpu.h>
16 #include <linux/cpu.h>
17 #include <linux/kthread.h>
18 #include <linux/rcupdate.h>
19 #include <linux/smp.h>
20 #include <linux/tick.h>
21 
22 #include <asm/irq.h>
23 /*
24    - No shared variables, all the data are CPU local.
25    - If a softirq needs serialization, let it serialize itself
26      by its own spinlocks.
27    - Even if softirq is serialized, only local cpu is marked for
28      execution. Hence, we get something sort of weak cpu binding.
29      Though it is still not clear, will it result in better locality
30      or will not.
31 
32    Examples:
33    - NET RX softirq. It is multithreaded and does not require
34      any global serialization.
35    - NET TX softirq. It kicks software netdevice queues, hence
36      it is logically serialized per device, but this serialization
37      is invisible to common code.
38    - Tasklets: serialized wrt itself.
39  */
40 
41 #ifndef __ARCH_IRQ_STAT
42 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
43 EXPORT_SYMBOL(irq_stat);
44 #endif
45 
46 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
47 
48 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
49 
50 /*
51  * we cannot loop indefinitely here to avoid userspace starvation,
52  * but we also don't want to introduce a worst case 1/HZ latency
53  * to the pending events, so lets the scheduler to balance
54  * the softirq load for us.
55  */
56 static inline void wakeup_softirqd(void)
57 {
58 	/* Interrupts are disabled: no need to stop preemption */
59 	struct task_struct *tsk = __get_cpu_var(ksoftirqd);
60 
61 	if (tsk && tsk->state != TASK_RUNNING)
62 		wake_up_process(tsk);
63 }
64 
65 /*
66  * This one is for softirq.c-internal use,
67  * where hardirqs are disabled legitimately:
68  */
69 #ifdef CONFIG_TRACE_IRQFLAGS
70 static void __local_bh_disable(unsigned long ip)
71 {
72 	unsigned long flags;
73 
74 	WARN_ON_ONCE(in_irq());
75 
76 	raw_local_irq_save(flags);
77 	add_preempt_count(SOFTIRQ_OFFSET);
78 	/*
79 	 * Were softirqs turned off above:
80 	 */
81 	if (softirq_count() == SOFTIRQ_OFFSET)
82 		trace_softirqs_off(ip);
83 	raw_local_irq_restore(flags);
84 }
85 #else /* !CONFIG_TRACE_IRQFLAGS */
86 static inline void __local_bh_disable(unsigned long ip)
87 {
88 	add_preempt_count(SOFTIRQ_OFFSET);
89 	barrier();
90 }
91 #endif /* CONFIG_TRACE_IRQFLAGS */
92 
93 void local_bh_disable(void)
94 {
95 	__local_bh_disable((unsigned long)__builtin_return_address(0));
96 }
97 
98 EXPORT_SYMBOL(local_bh_disable);
99 
100 void __local_bh_enable(void)
101 {
102 	WARN_ON_ONCE(in_irq());
103 
104 	/*
105 	 * softirqs should never be enabled by __local_bh_enable(),
106 	 * it always nests inside local_bh_enable() sections:
107 	 */
108 	WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
109 
110 	sub_preempt_count(SOFTIRQ_OFFSET);
111 }
112 EXPORT_SYMBOL_GPL(__local_bh_enable);
113 
114 /*
115  * Special-case - softirqs can safely be enabled in
116  * cond_resched_softirq(), or by __do_softirq(),
117  * without processing still-pending softirqs:
118  */
119 void _local_bh_enable(void)
120 {
121 	WARN_ON_ONCE(in_irq());
122 	WARN_ON_ONCE(!irqs_disabled());
123 
124 	if (softirq_count() == SOFTIRQ_OFFSET)
125 		trace_softirqs_on((unsigned long)__builtin_return_address(0));
126 	sub_preempt_count(SOFTIRQ_OFFSET);
127 }
128 
129 EXPORT_SYMBOL(_local_bh_enable);
130 
131 void local_bh_enable(void)
132 {
133 #ifdef CONFIG_TRACE_IRQFLAGS
134 	unsigned long flags;
135 
136 	WARN_ON_ONCE(in_irq());
137 #endif
138 	WARN_ON_ONCE(irqs_disabled());
139 
140 #ifdef CONFIG_TRACE_IRQFLAGS
141 	local_irq_save(flags);
142 #endif
143 	/*
144 	 * Are softirqs going to be turned on now:
145 	 */
146 	if (softirq_count() == SOFTIRQ_OFFSET)
147 		trace_softirqs_on((unsigned long)__builtin_return_address(0));
148 	/*
149 	 * Keep preemption disabled until we are done with
150 	 * softirq processing:
151  	 */
152  	sub_preempt_count(SOFTIRQ_OFFSET - 1);
153 
154 	if (unlikely(!in_interrupt() && local_softirq_pending()))
155 		do_softirq();
156 
157 	dec_preempt_count();
158 #ifdef CONFIG_TRACE_IRQFLAGS
159 	local_irq_restore(flags);
160 #endif
161 	preempt_check_resched();
162 }
163 EXPORT_SYMBOL(local_bh_enable);
164 
165 void local_bh_enable_ip(unsigned long ip)
166 {
167 #ifdef CONFIG_TRACE_IRQFLAGS
168 	unsigned long flags;
169 
170 	WARN_ON_ONCE(in_irq());
171 
172 	local_irq_save(flags);
173 #endif
174 	/*
175 	 * Are softirqs going to be turned on now:
176 	 */
177 	if (softirq_count() == SOFTIRQ_OFFSET)
178 		trace_softirqs_on(ip);
179 	/*
180 	 * Keep preemption disabled until we are done with
181 	 * softirq processing:
182  	 */
183  	sub_preempt_count(SOFTIRQ_OFFSET - 1);
184 
185 	if (unlikely(!in_interrupt() && local_softirq_pending()))
186 		do_softirq();
187 
188 	dec_preempt_count();
189 #ifdef CONFIG_TRACE_IRQFLAGS
190 	local_irq_restore(flags);
191 #endif
192 	preempt_check_resched();
193 }
194 EXPORT_SYMBOL(local_bh_enable_ip);
195 
196 /*
197  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
198  * and we fall back to softirqd after that.
199  *
200  * This number has been established via experimentation.
201  * The two things to balance is latency against fairness -
202  * we want to handle softirqs as soon as possible, but they
203  * should not be able to lock up the box.
204  */
205 #define MAX_SOFTIRQ_RESTART 10
206 
207 asmlinkage void __do_softirq(void)
208 {
209 	struct softirq_action *h;
210 	__u32 pending;
211 	int max_restart = MAX_SOFTIRQ_RESTART;
212 	int cpu;
213 
214 	pending = local_softirq_pending();
215 	account_system_vtime(current);
216 
217 	__local_bh_disable((unsigned long)__builtin_return_address(0));
218 	trace_softirq_enter();
219 
220 	cpu = smp_processor_id();
221 restart:
222 	/* Reset the pending bitmask before enabling irqs */
223 	set_softirq_pending(0);
224 
225 	local_irq_enable();
226 
227 	h = softirq_vec;
228 
229 	do {
230 		if (pending & 1) {
231 			h->action(h);
232 			rcu_bh_qsctr_inc(cpu);
233 		}
234 		h++;
235 		pending >>= 1;
236 	} while (pending);
237 
238 	local_irq_disable();
239 
240 	pending = local_softirq_pending();
241 	if (pending && --max_restart)
242 		goto restart;
243 
244 	if (pending)
245 		wakeup_softirqd();
246 
247 	trace_softirq_exit();
248 
249 	account_system_vtime(current);
250 	_local_bh_enable();
251 }
252 
253 #ifndef __ARCH_HAS_DO_SOFTIRQ
254 
255 asmlinkage void do_softirq(void)
256 {
257 	__u32 pending;
258 	unsigned long flags;
259 
260 	if (in_interrupt())
261 		return;
262 
263 	local_irq_save(flags);
264 
265 	pending = local_softirq_pending();
266 
267 	if (pending)
268 		__do_softirq();
269 
270 	local_irq_restore(flags);
271 }
272 
273 EXPORT_SYMBOL(do_softirq);
274 
275 #endif
276 
277 /*
278  * Enter an interrupt context.
279  */
280 void irq_enter(void)
281 {
282 	__irq_enter();
283 #ifdef CONFIG_NO_HZ
284 	if (idle_cpu(smp_processor_id()))
285 		tick_nohz_update_jiffies();
286 #endif
287 }
288 
289 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
290 # define invoke_softirq()	__do_softirq()
291 #else
292 # define invoke_softirq()	do_softirq()
293 #endif
294 
295 /*
296  * Exit an interrupt context. Process softirqs if needed and possible:
297  */
298 void irq_exit(void)
299 {
300 	account_system_vtime(current);
301 	trace_hardirq_exit();
302 	sub_preempt_count(IRQ_EXIT_OFFSET);
303 	if (!in_interrupt() && local_softirq_pending())
304 		invoke_softirq();
305 
306 #ifdef CONFIG_NO_HZ
307 	/* Make sure that timer wheel updates are propagated */
308 	if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
309 		tick_nohz_stop_sched_tick();
310 #endif
311 	preempt_enable_no_resched();
312 }
313 
314 /*
315  * This function must run with irqs disabled!
316  */
317 inline fastcall void raise_softirq_irqoff(unsigned int nr)
318 {
319 	__raise_softirq_irqoff(nr);
320 
321 	/*
322 	 * If we're in an interrupt or softirq, we're done
323 	 * (this also catches softirq-disabled code). We will
324 	 * actually run the softirq once we return from
325 	 * the irq or softirq.
326 	 *
327 	 * Otherwise we wake up ksoftirqd to make sure we
328 	 * schedule the softirq soon.
329 	 */
330 	if (!in_interrupt())
331 		wakeup_softirqd();
332 }
333 
334 EXPORT_SYMBOL(raise_softirq_irqoff);
335 
336 void fastcall raise_softirq(unsigned int nr)
337 {
338 	unsigned long flags;
339 
340 	local_irq_save(flags);
341 	raise_softirq_irqoff(nr);
342 	local_irq_restore(flags);
343 }
344 
345 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
346 {
347 	softirq_vec[nr].data = data;
348 	softirq_vec[nr].action = action;
349 }
350 
351 /* Tasklets */
352 struct tasklet_head
353 {
354 	struct tasklet_struct *list;
355 };
356 
357 /* Some compilers disobey section attribute on statics when not
358    initialized -- RR */
359 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
360 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
361 
362 void fastcall __tasklet_schedule(struct tasklet_struct *t)
363 {
364 	unsigned long flags;
365 
366 	local_irq_save(flags);
367 	t->next = __get_cpu_var(tasklet_vec).list;
368 	__get_cpu_var(tasklet_vec).list = t;
369 	raise_softirq_irqoff(TASKLET_SOFTIRQ);
370 	local_irq_restore(flags);
371 }
372 
373 EXPORT_SYMBOL(__tasklet_schedule);
374 
375 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
376 {
377 	unsigned long flags;
378 
379 	local_irq_save(flags);
380 	t->next = __get_cpu_var(tasklet_hi_vec).list;
381 	__get_cpu_var(tasklet_hi_vec).list = t;
382 	raise_softirq_irqoff(HI_SOFTIRQ);
383 	local_irq_restore(flags);
384 }
385 
386 EXPORT_SYMBOL(__tasklet_hi_schedule);
387 
388 static void tasklet_action(struct softirq_action *a)
389 {
390 	struct tasklet_struct *list;
391 
392 	local_irq_disable();
393 	list = __get_cpu_var(tasklet_vec).list;
394 	__get_cpu_var(tasklet_vec).list = NULL;
395 	local_irq_enable();
396 
397 	while (list) {
398 		struct tasklet_struct *t = list;
399 
400 		list = list->next;
401 
402 		if (tasklet_trylock(t)) {
403 			if (!atomic_read(&t->count)) {
404 				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
405 					BUG();
406 				t->func(t->data);
407 				tasklet_unlock(t);
408 				continue;
409 			}
410 			tasklet_unlock(t);
411 		}
412 
413 		local_irq_disable();
414 		t->next = __get_cpu_var(tasklet_vec).list;
415 		__get_cpu_var(tasklet_vec).list = t;
416 		__raise_softirq_irqoff(TASKLET_SOFTIRQ);
417 		local_irq_enable();
418 	}
419 }
420 
421 static void tasklet_hi_action(struct softirq_action *a)
422 {
423 	struct tasklet_struct *list;
424 
425 	local_irq_disable();
426 	list = __get_cpu_var(tasklet_hi_vec).list;
427 	__get_cpu_var(tasklet_hi_vec).list = NULL;
428 	local_irq_enable();
429 
430 	while (list) {
431 		struct tasklet_struct *t = list;
432 
433 		list = list->next;
434 
435 		if (tasklet_trylock(t)) {
436 			if (!atomic_read(&t->count)) {
437 				if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
438 					BUG();
439 				t->func(t->data);
440 				tasklet_unlock(t);
441 				continue;
442 			}
443 			tasklet_unlock(t);
444 		}
445 
446 		local_irq_disable();
447 		t->next = __get_cpu_var(tasklet_hi_vec).list;
448 		__get_cpu_var(tasklet_hi_vec).list = t;
449 		__raise_softirq_irqoff(HI_SOFTIRQ);
450 		local_irq_enable();
451 	}
452 }
453 
454 
455 void tasklet_init(struct tasklet_struct *t,
456 		  void (*func)(unsigned long), unsigned long data)
457 {
458 	t->next = NULL;
459 	t->state = 0;
460 	atomic_set(&t->count, 0);
461 	t->func = func;
462 	t->data = data;
463 }
464 
465 EXPORT_SYMBOL(tasklet_init);
466 
467 void tasklet_kill(struct tasklet_struct *t)
468 {
469 	if (in_interrupt())
470 		printk("Attempt to kill tasklet from interrupt\n");
471 
472 	while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
473 		do
474 			yield();
475 		while (test_bit(TASKLET_STATE_SCHED, &t->state));
476 	}
477 	tasklet_unlock_wait(t);
478 	clear_bit(TASKLET_STATE_SCHED, &t->state);
479 }
480 
481 EXPORT_SYMBOL(tasklet_kill);
482 
483 void __init softirq_init(void)
484 {
485 	open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
486 	open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
487 }
488 
489 static int ksoftirqd(void * __bind_cpu)
490 {
491 	set_user_nice(current, 19);
492 	current->flags |= PF_NOFREEZE;
493 
494 	set_current_state(TASK_INTERRUPTIBLE);
495 
496 	while (!kthread_should_stop()) {
497 		preempt_disable();
498 		if (!local_softirq_pending()) {
499 			preempt_enable_no_resched();
500 			schedule();
501 			preempt_disable();
502 		}
503 
504 		__set_current_state(TASK_RUNNING);
505 
506 		while (local_softirq_pending()) {
507 			/* Preempt disable stops cpu going offline.
508 			   If already offline, we'll be on wrong CPU:
509 			   don't process */
510 			if (cpu_is_offline((long)__bind_cpu))
511 				goto wait_to_die;
512 			do_softirq();
513 			preempt_enable_no_resched();
514 			cond_resched();
515 			preempt_disable();
516 		}
517 		preempt_enable();
518 		set_current_state(TASK_INTERRUPTIBLE);
519 	}
520 	__set_current_state(TASK_RUNNING);
521 	return 0;
522 
523 wait_to_die:
524 	preempt_enable();
525 	/* Wait for kthread_stop */
526 	set_current_state(TASK_INTERRUPTIBLE);
527 	while (!kthread_should_stop()) {
528 		schedule();
529 		set_current_state(TASK_INTERRUPTIBLE);
530 	}
531 	__set_current_state(TASK_RUNNING);
532 	return 0;
533 }
534 
535 #ifdef CONFIG_HOTPLUG_CPU
536 /*
537  * tasklet_kill_immediate is called to remove a tasklet which can already be
538  * scheduled for execution on @cpu.
539  *
540  * Unlike tasklet_kill, this function removes the tasklet
541  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
542  *
543  * When this function is called, @cpu must be in the CPU_DEAD state.
544  */
545 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
546 {
547 	struct tasklet_struct **i;
548 
549 	BUG_ON(cpu_online(cpu));
550 	BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
551 
552 	if (!test_bit(TASKLET_STATE_SCHED, &t->state))
553 		return;
554 
555 	/* CPU is dead, so no lock needed. */
556 	for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
557 		if (*i == t) {
558 			*i = t->next;
559 			return;
560 		}
561 	}
562 	BUG();
563 }
564 
565 static void takeover_tasklets(unsigned int cpu)
566 {
567 	struct tasklet_struct **i;
568 
569 	/* CPU is dead, so no lock needed. */
570 	local_irq_disable();
571 
572 	/* Find end, append list for that CPU. */
573 	for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
574 	*i = per_cpu(tasklet_vec, cpu).list;
575 	per_cpu(tasklet_vec, cpu).list = NULL;
576 	raise_softirq_irqoff(TASKLET_SOFTIRQ);
577 
578 	for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
579 	*i = per_cpu(tasklet_hi_vec, cpu).list;
580 	per_cpu(tasklet_hi_vec, cpu).list = NULL;
581 	raise_softirq_irqoff(HI_SOFTIRQ);
582 
583 	local_irq_enable();
584 }
585 #endif /* CONFIG_HOTPLUG_CPU */
586 
587 static int __cpuinit cpu_callback(struct notifier_block *nfb,
588 				  unsigned long action,
589 				  void *hcpu)
590 {
591 	int hotcpu = (unsigned long)hcpu;
592 	struct task_struct *p;
593 
594 	switch (action) {
595 	case CPU_UP_PREPARE:
596 	case CPU_UP_PREPARE_FROZEN:
597 		p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
598 		if (IS_ERR(p)) {
599 			printk("ksoftirqd for %i failed\n", hotcpu);
600 			return NOTIFY_BAD;
601 		}
602 		kthread_bind(p, hotcpu);
603   		per_cpu(ksoftirqd, hotcpu) = p;
604  		break;
605 	case CPU_ONLINE:
606 	case CPU_ONLINE_FROZEN:
607 		wake_up_process(per_cpu(ksoftirqd, hotcpu));
608 		break;
609 #ifdef CONFIG_HOTPLUG_CPU
610 	case CPU_UP_CANCELED:
611 	case CPU_UP_CANCELED_FROZEN:
612 		if (!per_cpu(ksoftirqd, hotcpu))
613 			break;
614 		/* Unbind so it can run.  Fall thru. */
615 		kthread_bind(per_cpu(ksoftirqd, hotcpu),
616 			     any_online_cpu(cpu_online_map));
617 	case CPU_DEAD:
618 	case CPU_DEAD_FROZEN:
619 		p = per_cpu(ksoftirqd, hotcpu);
620 		per_cpu(ksoftirqd, hotcpu) = NULL;
621 		kthread_stop(p);
622 		takeover_tasklets(hotcpu);
623 		break;
624 #endif /* CONFIG_HOTPLUG_CPU */
625  	}
626 	return NOTIFY_OK;
627 }
628 
629 static struct notifier_block __cpuinitdata cpu_nfb = {
630 	.notifier_call = cpu_callback
631 };
632 
633 __init int spawn_ksoftirqd(void)
634 {
635 	void *cpu = (void *)(long)smp_processor_id();
636 	int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
637 
638 	BUG_ON(err == NOTIFY_BAD);
639 	cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
640 	register_cpu_notifier(&cpu_nfb);
641 	return 0;
642 }
643 
644 #ifdef CONFIG_SMP
645 /*
646  * Call a function on all processors
647  */
648 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
649 {
650 	int ret = 0;
651 
652 	preempt_disable();
653 	ret = smp_call_function(func, info, retry, wait);
654 	local_irq_disable();
655 	func(info);
656 	local_irq_enable();
657 	preempt_enable();
658 	return ret;
659 }
660 EXPORT_SYMBOL(on_each_cpu);
661 #endif
662