xref: /openbmc/linux/arch/powerpc/kernel/irq.c (revision f5ad1c74)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  Derived from arch/i386/kernel/irq.c
4  *    Copyright (C) 1992 Linus Torvalds
5  *  Adapted from arch/i386 by Gary Thomas
6  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
7  *  Updated and modified by Cort Dougan <cort@fsmlabs.com>
8  *    Copyright (C) 1996-2001 Cort Dougan
9  *  Adapted for Power Macintosh by Paul Mackerras
10  *    Copyright (C) 1996 Paul Mackerras (paulus@cs.anu.edu.au)
11  *
12  * This file contains the code used by various IRQ handling routines:
13  * asking for different IRQ's should be done through these routines
14  * instead of just grabbing them. Thus setups with different IRQ numbers
15  * shouldn't result in any weird surprises, and installing new handlers
16  * should be easier.
17  *
18  * The MPC8xx has an interrupt mask in the SIU.  If a bit is set, the
19  * interrupt is _enabled_.  As expected, IRQ0 is bit 0 in the 32-bit
20  * mask register (of which only 16 are defined), hence the weird shifting
21  * and complement of the cached_irq_mask.  I want to be able to stuff
22  * this right into the SIU SMASK register.
23  * Many of the prep/chrp functions are conditional compiled on CONFIG_PPC_8xx
24  * to reduce code space and undefined function references.
25  */
26 
27 #undef DEBUG
28 
29 #include <linux/export.h>
30 #include <linux/threads.h>
31 #include <linux/kernel_stat.h>
32 #include <linux/signal.h>
33 #include <linux/sched.h>
34 #include <linux/ptrace.h>
35 #include <linux/ioport.h>
36 #include <linux/interrupt.h>
37 #include <linux/timex.h>
38 #include <linux/init.h>
39 #include <linux/slab.h>
40 #include <linux/delay.h>
41 #include <linux/irq.h>
42 #include <linux/seq_file.h>
43 #include <linux/cpumask.h>
44 #include <linux/profile.h>
45 #include <linux/bitops.h>
46 #include <linux/list.h>
47 #include <linux/radix-tree.h>
48 #include <linux/mutex.h>
49 #include <linux/pci.h>
50 #include <linux/debugfs.h>
51 #include <linux/of.h>
52 #include <linux/of_irq.h>
53 #include <linux/vmalloc.h>
54 #include <linux/pgtable.h>
55 
56 #include <linux/uaccess.h>
57 #include <asm/io.h>
58 #include <asm/irq.h>
59 #include <asm/cache.h>
60 #include <asm/prom.h>
61 #include <asm/ptrace.h>
62 #include <asm/machdep.h>
63 #include <asm/udbg.h>
64 #include <asm/smp.h>
65 #include <asm/livepatch.h>
66 #include <asm/asm-prototypes.h>
67 #include <asm/hw_irq.h>
68 
69 #ifdef CONFIG_PPC64
70 #include <asm/paca.h>
71 #include <asm/firmware.h>
72 #include <asm/lv1call.h>
73 #include <asm/dbell.h>
74 #endif
75 #define CREATE_TRACE_POINTS
76 #include <asm/trace.h>
77 #include <asm/cpu_has_feature.h>
78 
79 DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
80 EXPORT_PER_CPU_SYMBOL(irq_stat);
81 
82 #ifdef CONFIG_PPC32
83 atomic_t ppc_n_lost_interrupts;
84 
85 #ifdef CONFIG_TAU_INT
86 extern int tau_initialized;
87 u32 tau_interrupts(unsigned long cpu);
88 #endif
89 #endif /* CONFIG_PPC32 */
90 
91 #ifdef CONFIG_PPC64
92 
93 int distribute_irqs = 1;
94 
95 static inline notrace unsigned long get_irq_happened(void)
96 {
97 	unsigned long happened;
98 
99 	__asm__ __volatile__("lbz %0,%1(13)"
100 	: "=r" (happened) : "i" (offsetof(struct paca_struct, irq_happened)));
101 
102 	return happened;
103 }
104 
105 static inline notrace int decrementer_check_overflow(void)
106 {
107 	u64 now = get_tb();
108 	u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
109 
110 	return now >= *next_tb;
111 }
112 
113 #ifdef CONFIG_PPC_BOOK3E
114 
115 /* This is called whenever we are re-enabling interrupts
116  * and returns either 0 (nothing to do) or 500/900/280 if
117  * there's an EE, DEC or DBELL to generate.
118  *
119  * This is called in two contexts: From arch_local_irq_restore()
120  * before soft-enabling interrupts, and from the exception exit
121  * path when returning from an interrupt from a soft-disabled to
122  * a soft enabled context. In both case we have interrupts hard
123  * disabled.
124  *
125  * We take care of only clearing the bits we handled in the
126  * PACA irq_happened field since we can only re-emit one at a
127  * time and we don't want to "lose" one.
128  */
129 notrace unsigned int __check_irq_replay(void)
130 {
131 	/*
132 	 * We use local_paca rather than get_paca() to avoid all
133 	 * the debug_smp_processor_id() business in this low level
134 	 * function
135 	 */
136 	unsigned char happened = local_paca->irq_happened;
137 
138 	/*
139 	 * We are responding to the next interrupt, so interrupt-off
140 	 * latencies should be reset here.
141 	 */
142 	trace_hardirqs_on();
143 	trace_hardirqs_off();
144 
145 	/*
146 	 * We are always hard disabled here, but PACA_IRQ_HARD_DIS may
147 	 * not be set, which means interrupts have only just been hard
148 	 * disabled as part of the local_irq_restore or interrupt return
149 	 * code. In that case, skip the decrementr check becaus it's
150 	 * expensive to read the TB.
151 	 *
152 	 * HARD_DIS then gets cleared here, but it's reconciled later.
153 	 * Either local_irq_disable will replay the interrupt and that
154 	 * will reconcile state like other hard interrupts. Or interrupt
155 	 * retur will replay the interrupt and in that case it sets
156 	 * PACA_IRQ_HARD_DIS by hand (see comments in entry_64.S).
157 	 */
158 	if (happened & PACA_IRQ_HARD_DIS) {
159 		local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
160 
161 		/*
162 		 * We may have missed a decrementer interrupt if hard disabled.
163 		 * Check the decrementer register in case we had a rollover
164 		 * while hard disabled.
165 		 */
166 		if (!(happened & PACA_IRQ_DEC)) {
167 			if (decrementer_check_overflow()) {
168 				local_paca->irq_happened |= PACA_IRQ_DEC;
169 				happened |= PACA_IRQ_DEC;
170 			}
171 		}
172 	}
173 
174 	if (happened & PACA_IRQ_DEC) {
175 		local_paca->irq_happened &= ~PACA_IRQ_DEC;
176 		return 0x900;
177 	}
178 
179 	if (happened & PACA_IRQ_EE) {
180 		local_paca->irq_happened &= ~PACA_IRQ_EE;
181 		return 0x500;
182 	}
183 
184 	if (happened & PACA_IRQ_DBELL) {
185 		local_paca->irq_happened &= ~PACA_IRQ_DBELL;
186 		return 0x280;
187 	}
188 
189 	/* There should be nothing left ! */
190 	BUG_ON(local_paca->irq_happened != 0);
191 
192 	return 0;
193 }
194 
195 /*
196  * This is specifically called by assembly code to re-enable interrupts
197  * if they are currently disabled. This is typically called before
198  * schedule() or do_signal() when returning to userspace. We do it
199  * in C to avoid the burden of dealing with lockdep etc...
200  *
201  * NOTE: This is called with interrupts hard disabled but not marked
202  * as such in paca->irq_happened, so we need to resync this.
203  */
204 void notrace restore_interrupts(void)
205 {
206 	if (irqs_disabled()) {
207 		local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
208 		local_irq_enable();
209 	} else
210 		__hard_irq_enable();
211 }
212 
213 #endif /* CONFIG_PPC_BOOK3E */
214 
215 void replay_soft_interrupts(void)
216 {
217 	/*
218 	 * We use local_paca rather than get_paca() to avoid all
219 	 * the debug_smp_processor_id() business in this low level
220 	 * function
221 	 */
222 	unsigned char happened = local_paca->irq_happened;
223 	struct pt_regs regs;
224 
225 	ppc_save_regs(&regs);
226 	regs.softe = IRQS_ENABLED;
227 
228 again:
229 	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
230 		WARN_ON_ONCE(mfmsr() & MSR_EE);
231 
232 	if (happened & PACA_IRQ_HARD_DIS) {
233 		/*
234 		 * We may have missed a decrementer interrupt if hard disabled.
235 		 * Check the decrementer register in case we had a rollover
236 		 * while hard disabled.
237 		 */
238 		if (!(happened & PACA_IRQ_DEC)) {
239 			if (decrementer_check_overflow())
240 				happened |= PACA_IRQ_DEC;
241 		}
242 	}
243 
244 	/*
245 	 * Force the delivery of pending soft-disabled interrupts on PS3.
246 	 * Any HV call will have this side effect.
247 	 */
248 	if (firmware_has_feature(FW_FEATURE_PS3_LV1)) {
249 		u64 tmp, tmp2;
250 		lv1_get_version_info(&tmp, &tmp2);
251 	}
252 
253 	/*
254 	 * Check if an hypervisor Maintenance interrupt happened.
255 	 * This is a higher priority interrupt than the others, so
256 	 * replay it first.
257 	 */
258 	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (happened & PACA_IRQ_HMI)) {
259 		local_paca->irq_happened &= ~PACA_IRQ_HMI;
260 		regs.trap = 0xe60;
261 		handle_hmi_exception(&regs);
262 		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
263 			hard_irq_disable();
264 	}
265 
266 	if (happened & PACA_IRQ_DEC) {
267 		local_paca->irq_happened &= ~PACA_IRQ_DEC;
268 		regs.trap = 0x900;
269 		timer_interrupt(&regs);
270 		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
271 			hard_irq_disable();
272 	}
273 
274 	if (happened & PACA_IRQ_EE) {
275 		local_paca->irq_happened &= ~PACA_IRQ_EE;
276 		regs.trap = 0x500;
277 		do_IRQ(&regs);
278 		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
279 			hard_irq_disable();
280 	}
281 
282 	if (IS_ENABLED(CONFIG_PPC_DOORBELL) && (happened & PACA_IRQ_DBELL)) {
283 		local_paca->irq_happened &= ~PACA_IRQ_DBELL;
284 		if (IS_ENABLED(CONFIG_PPC_BOOK3E))
285 			regs.trap = 0x280;
286 		else
287 			regs.trap = 0xa00;
288 		doorbell_exception(&regs);
289 		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
290 			hard_irq_disable();
291 	}
292 
293 	/* Book3E does not support soft-masking PMI interrupts */
294 	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (happened & PACA_IRQ_PMI)) {
295 		local_paca->irq_happened &= ~PACA_IRQ_PMI;
296 		regs.trap = 0xf00;
297 		performance_monitor_exception(&regs);
298 		if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
299 			hard_irq_disable();
300 	}
301 
302 	happened = local_paca->irq_happened;
303 	if (happened & ~PACA_IRQ_HARD_DIS) {
304 		/*
305 		 * We are responding to the next interrupt, so interrupt-off
306 		 * latencies should be reset here.
307 		 */
308 		trace_hardirqs_on();
309 		trace_hardirqs_off();
310 		goto again;
311 	}
312 }
313 
314 notrace void arch_local_irq_restore(unsigned long mask)
315 {
316 	unsigned char irq_happened;
317 
318 	/* Write the new soft-enabled value */
319 	irq_soft_mask_set(mask);
320 	if (mask)
321 		return;
322 
323 	/*
324 	 * From this point onward, we can take interrupts, preempt,
325 	 * etc... unless we got hard-disabled. We check if an event
326 	 * happened. If none happened, we know we can just return.
327 	 *
328 	 * We may have preempted before the check below, in which case
329 	 * we are checking the "new" CPU instead of the old one. This
330 	 * is only a problem if an event happened on the "old" CPU.
331 	 *
332 	 * External interrupt events will have caused interrupts to
333 	 * be hard-disabled, so there is no problem, we
334 	 * cannot have preempted.
335 	 */
336 	irq_happened = get_irq_happened();
337 	if (!irq_happened) {
338 		if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
339 			WARN_ON_ONCE(!(mfmsr() & MSR_EE));
340 		return;
341 	}
342 
343 	/* We need to hard disable to replay. */
344 	if (!(irq_happened & PACA_IRQ_HARD_DIS)) {
345 		if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
346 			WARN_ON_ONCE(!(mfmsr() & MSR_EE));
347 		__hard_irq_disable();
348 	} else {
349 		/*
350 		 * We should already be hard disabled here. We had bugs
351 		 * where that wasn't the case so let's dbl check it and
352 		 * warn if we are wrong. Only do that when IRQ tracing
353 		 * is enabled as mfmsr() can be costly.
354 		 */
355 		if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
356 			if (WARN_ON_ONCE(mfmsr() & MSR_EE))
357 				__hard_irq_disable();
358 		}
359 
360 		if (irq_happened == PACA_IRQ_HARD_DIS) {
361 			local_paca->irq_happened = 0;
362 			__hard_irq_enable();
363 			return;
364 		}
365 	}
366 
367 	/*
368 	 * Disable preempt here, so that the below preempt_enable will
369 	 * perform resched if required (a replayed interrupt may set
370 	 * need_resched).
371 	 */
372 	preempt_disable();
373 	irq_soft_mask_set(IRQS_ALL_DISABLED);
374 	trace_hardirqs_off();
375 
376 	replay_soft_interrupts();
377 	local_paca->irq_happened = 0;
378 
379 	trace_hardirqs_on();
380 	irq_soft_mask_set(IRQS_ENABLED);
381 	__hard_irq_enable();
382 	preempt_enable();
383 }
384 EXPORT_SYMBOL(arch_local_irq_restore);
385 
386 /*
387  * This is a helper to use when about to go into idle low-power
388  * when the latter has the side effect of re-enabling interrupts
389  * (such as calling H_CEDE under pHyp).
390  *
391  * You call this function with interrupts soft-disabled (this is
392  * already the case when ppc_md.power_save is called). The function
393  * will return whether to enter power save or just return.
394  *
395  * In the former case, it will have notified lockdep of interrupts
396  * being re-enabled and generally sanitized the lazy irq state,
397  * and in the latter case it will leave with interrupts hard
398  * disabled and marked as such, so the local_irq_enable() call
399  * in arch_cpu_idle() will properly re-enable everything.
400  */
401 bool prep_irq_for_idle(void)
402 {
403 	/*
404 	 * First we need to hard disable to ensure no interrupt
405 	 * occurs before we effectively enter the low power state
406 	 */
407 	__hard_irq_disable();
408 	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
409 
410 	/*
411 	 * If anything happened while we were soft-disabled,
412 	 * we return now and do not enter the low power state.
413 	 */
414 	if (lazy_irq_pending())
415 		return false;
416 
417 	/* Tell lockdep we are about to re-enable */
418 	trace_hardirqs_on();
419 
420 	/*
421 	 * Mark interrupts as soft-enabled and clear the
422 	 * PACA_IRQ_HARD_DIS from the pending mask since we
423 	 * are about to hard enable as well as a side effect
424 	 * of entering the low power state.
425 	 */
426 	local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
427 	irq_soft_mask_set(IRQS_ENABLED);
428 
429 	/* Tell the caller to enter the low power state */
430 	return true;
431 }
432 
433 #ifdef CONFIG_PPC_BOOK3S
434 /*
435  * This is for idle sequences that return with IRQs off, but the
436  * idle state itself wakes on interrupt. Tell the irq tracer that
437  * IRQs are enabled for the duration of idle so it does not get long
438  * off times. Must be paired with fini_irq_for_idle_irqsoff.
439  */
440 bool prep_irq_for_idle_irqsoff(void)
441 {
442 	WARN_ON(!irqs_disabled());
443 
444 	/*
445 	 * First we need to hard disable to ensure no interrupt
446 	 * occurs before we effectively enter the low power state
447 	 */
448 	__hard_irq_disable();
449 	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
450 
451 	/*
452 	 * If anything happened while we were soft-disabled,
453 	 * we return now and do not enter the low power state.
454 	 */
455 	if (lazy_irq_pending())
456 		return false;
457 
458 	/* Tell lockdep we are about to re-enable */
459 	trace_hardirqs_on();
460 
461 	return true;
462 }
463 
464 /*
465  * Take the SRR1 wakeup reason, index into this table to find the
466  * appropriate irq_happened bit.
467  *
468  * Sytem reset exceptions taken in idle state also come through here,
469  * but they are NMI interrupts so do not need to wait for IRQs to be
470  * restored, and should be taken as early as practical. These are marked
471  * with 0xff in the table. The Power ISA specifies 0100b as the system
472  * reset interrupt reason.
473  */
474 #define IRQ_SYSTEM_RESET	0xff
475 
476 static const u8 srr1_to_lazyirq[0x10] = {
477 	0, 0, 0,
478 	PACA_IRQ_DBELL,
479 	IRQ_SYSTEM_RESET,
480 	PACA_IRQ_DBELL,
481 	PACA_IRQ_DEC,
482 	0,
483 	PACA_IRQ_EE,
484 	PACA_IRQ_EE,
485 	PACA_IRQ_HMI,
486 	0, 0, 0, 0, 0 };
487 
488 void replay_system_reset(void)
489 {
490 	struct pt_regs regs;
491 
492 	ppc_save_regs(&regs);
493 	regs.trap = 0x100;
494 	get_paca()->in_nmi = 1;
495 	system_reset_exception(&regs);
496 	get_paca()->in_nmi = 0;
497 }
498 EXPORT_SYMBOL_GPL(replay_system_reset);
499 
500 void irq_set_pending_from_srr1(unsigned long srr1)
501 {
502 	unsigned int idx = (srr1 & SRR1_WAKEMASK_P8) >> 18;
503 	u8 reason = srr1_to_lazyirq[idx];
504 
505 	/*
506 	 * Take the system reset now, which is immediately after registers
507 	 * are restored from idle. It's an NMI, so interrupts need not be
508 	 * re-enabled before it is taken.
509 	 */
510 	if (unlikely(reason == IRQ_SYSTEM_RESET)) {
511 		replay_system_reset();
512 		return;
513 	}
514 
515 	if (reason == PACA_IRQ_DBELL) {
516 		/*
517 		 * When doorbell triggers a system reset wakeup, the message
518 		 * is not cleared, so if the doorbell interrupt is replayed
519 		 * and the IPI handled, the doorbell interrupt would still
520 		 * fire when EE is enabled.
521 		 *
522 		 * To avoid taking the superfluous doorbell interrupt,
523 		 * execute a msgclr here before the interrupt is replayed.
524 		 */
525 		ppc_msgclr(PPC_DBELL_MSGTYPE);
526 	}
527 
528 	/*
529 	 * The 0 index (SRR1[42:45]=b0000) must always evaluate to 0,
530 	 * so this can be called unconditionally with the SRR1 wake
531 	 * reason as returned by the idle code, which uses 0 to mean no
532 	 * interrupt.
533 	 *
534 	 * If a future CPU was to designate this as an interrupt reason,
535 	 * then a new index for no interrupt must be assigned.
536 	 */
537 	local_paca->irq_happened |= reason;
538 }
539 #endif /* CONFIG_PPC_BOOK3S */
540 
541 /*
542  * Force a replay of the external interrupt handler on this CPU.
543  */
544 void force_external_irq_replay(void)
545 {
546 	/*
547 	 * This must only be called with interrupts soft-disabled,
548 	 * the replay will happen when re-enabling.
549 	 */
550 	WARN_ON(!arch_irqs_disabled());
551 
552 	/*
553 	 * Interrupts must always be hard disabled before irq_happened is
554 	 * modified (to prevent lost update in case of interrupt between
555 	 * load and store).
556 	 */
557 	__hard_irq_disable();
558 	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
559 
560 	/* Indicate in the PACA that we have an interrupt to replay */
561 	local_paca->irq_happened |= PACA_IRQ_EE;
562 }
563 
564 #endif /* CONFIG_PPC64 */
565 
566 int arch_show_interrupts(struct seq_file *p, int prec)
567 {
568 	int j;
569 
570 #if defined(CONFIG_PPC32) && defined(CONFIG_TAU_INT)
571 	if (tau_initialized) {
572 		seq_printf(p, "%*s: ", prec, "TAU");
573 		for_each_online_cpu(j)
574 			seq_printf(p, "%10u ", tau_interrupts(j));
575 		seq_puts(p, "  PowerPC             Thermal Assist (cpu temp)\n");
576 	}
577 #endif /* CONFIG_PPC32 && CONFIG_TAU_INT */
578 
579 	seq_printf(p, "%*s: ", prec, "LOC");
580 	for_each_online_cpu(j)
581 		seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs_event);
582         seq_printf(p, "  Local timer interrupts for timer event device\n");
583 
584 	seq_printf(p, "%*s: ", prec, "BCT");
585 	for_each_online_cpu(j)
586 		seq_printf(p, "%10u ", per_cpu(irq_stat, j).broadcast_irqs_event);
587 	seq_printf(p, "  Broadcast timer interrupts for timer event device\n");
588 
589 	seq_printf(p, "%*s: ", prec, "LOC");
590 	for_each_online_cpu(j)
591 		seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs_others);
592         seq_printf(p, "  Local timer interrupts for others\n");
593 
594 	seq_printf(p, "%*s: ", prec, "SPU");
595 	for_each_online_cpu(j)
596 		seq_printf(p, "%10u ", per_cpu(irq_stat, j).spurious_irqs);
597 	seq_printf(p, "  Spurious interrupts\n");
598 
599 	seq_printf(p, "%*s: ", prec, "PMI");
600 	for_each_online_cpu(j)
601 		seq_printf(p, "%10u ", per_cpu(irq_stat, j).pmu_irqs);
602 	seq_printf(p, "  Performance monitoring interrupts\n");
603 
604 	seq_printf(p, "%*s: ", prec, "MCE");
605 	for_each_online_cpu(j)
606 		seq_printf(p, "%10u ", per_cpu(irq_stat, j).mce_exceptions);
607 	seq_printf(p, "  Machine check exceptions\n");
608 
609 #ifdef CONFIG_PPC_BOOK3S_64
610 	if (cpu_has_feature(CPU_FTR_HVMODE)) {
611 		seq_printf(p, "%*s: ", prec, "HMI");
612 		for_each_online_cpu(j)
613 			seq_printf(p, "%10u ", paca_ptrs[j]->hmi_irqs);
614 		seq_printf(p, "  Hypervisor Maintenance Interrupts\n");
615 	}
616 #endif
617 
618 	seq_printf(p, "%*s: ", prec, "NMI");
619 	for_each_online_cpu(j)
620 		seq_printf(p, "%10u ", per_cpu(irq_stat, j).sreset_irqs);
621 	seq_printf(p, "  System Reset interrupts\n");
622 
623 #ifdef CONFIG_PPC_WATCHDOG
624 	seq_printf(p, "%*s: ", prec, "WDG");
625 	for_each_online_cpu(j)
626 		seq_printf(p, "%10u ", per_cpu(irq_stat, j).soft_nmi_irqs);
627 	seq_printf(p, "  Watchdog soft-NMI interrupts\n");
628 #endif
629 
630 #ifdef CONFIG_PPC_DOORBELL
631 	if (cpu_has_feature(CPU_FTR_DBELL)) {
632 		seq_printf(p, "%*s: ", prec, "DBL");
633 		for_each_online_cpu(j)
634 			seq_printf(p, "%10u ", per_cpu(irq_stat, j).doorbell_irqs);
635 		seq_printf(p, "  Doorbell interrupts\n");
636 	}
637 #endif
638 
639 	return 0;
640 }
641 
642 /*
643  * /proc/stat helpers
644  */
645 u64 arch_irq_stat_cpu(unsigned int cpu)
646 {
647 	u64 sum = per_cpu(irq_stat, cpu).timer_irqs_event;
648 
649 	sum += per_cpu(irq_stat, cpu).broadcast_irqs_event;
650 	sum += per_cpu(irq_stat, cpu).pmu_irqs;
651 	sum += per_cpu(irq_stat, cpu).mce_exceptions;
652 	sum += per_cpu(irq_stat, cpu).spurious_irqs;
653 	sum += per_cpu(irq_stat, cpu).timer_irqs_others;
654 #ifdef CONFIG_PPC_BOOK3S_64
655 	sum += paca_ptrs[cpu]->hmi_irqs;
656 #endif
657 	sum += per_cpu(irq_stat, cpu).sreset_irqs;
658 #ifdef CONFIG_PPC_WATCHDOG
659 	sum += per_cpu(irq_stat, cpu).soft_nmi_irqs;
660 #endif
661 #ifdef CONFIG_PPC_DOORBELL
662 	sum += per_cpu(irq_stat, cpu).doorbell_irqs;
663 #endif
664 
665 	return sum;
666 }
667 
668 static inline void check_stack_overflow(void)
669 {
670 	long sp;
671 
672 	if (!IS_ENABLED(CONFIG_DEBUG_STACKOVERFLOW))
673 		return;
674 
675 	sp = current_stack_pointer & (THREAD_SIZE - 1);
676 
677 	/* check for stack overflow: is there less than 2KB free? */
678 	if (unlikely(sp < 2048)) {
679 		pr_err("do_IRQ: stack overflow: %ld\n", sp);
680 		dump_stack();
681 	}
682 }
683 
684 void __do_irq(struct pt_regs *regs)
685 {
686 	unsigned int irq;
687 
688 	irq_enter();
689 
690 	trace_irq_entry(regs);
691 
692 	/*
693 	 * Query the platform PIC for the interrupt & ack it.
694 	 *
695 	 * This will typically lower the interrupt line to the CPU
696 	 */
697 	irq = ppc_md.get_irq();
698 
699 	/* We can hard enable interrupts now to allow perf interrupts */
700 	may_hard_irq_enable();
701 
702 	/* And finally process it */
703 	if (unlikely(!irq))
704 		__this_cpu_inc(irq_stat.spurious_irqs);
705 	else
706 		generic_handle_irq(irq);
707 
708 	trace_irq_exit(regs);
709 
710 	irq_exit();
711 }
712 
713 void do_IRQ(struct pt_regs *regs)
714 {
715 	struct pt_regs *old_regs = set_irq_regs(regs);
716 	void *cursp, *irqsp, *sirqsp;
717 
718 	/* Switch to the irq stack to handle this */
719 	cursp = (void *)(current_stack_pointer & ~(THREAD_SIZE - 1));
720 	irqsp = hardirq_ctx[raw_smp_processor_id()];
721 	sirqsp = softirq_ctx[raw_smp_processor_id()];
722 
723 	check_stack_overflow();
724 
725 	/* Already there ? */
726 	if (unlikely(cursp == irqsp || cursp == sirqsp)) {
727 		__do_irq(regs);
728 		set_irq_regs(old_regs);
729 		return;
730 	}
731 	/* Switch stack and call */
732 	call_do_irq(regs, irqsp);
733 
734 	set_irq_regs(old_regs);
735 }
736 
737 static void *__init alloc_vm_stack(void)
738 {
739 	return __vmalloc_node(THREAD_SIZE, THREAD_ALIGN, THREADINFO_GFP,
740 			      NUMA_NO_NODE, (void *)_RET_IP_);
741 }
742 
743 static void __init vmap_irqstack_init(void)
744 {
745 	int i;
746 
747 	for_each_possible_cpu(i) {
748 		softirq_ctx[i] = alloc_vm_stack();
749 		hardirq_ctx[i] = alloc_vm_stack();
750 	}
751 }
752 
753 
754 void __init init_IRQ(void)
755 {
756 	if (IS_ENABLED(CONFIG_VMAP_STACK))
757 		vmap_irqstack_init();
758 
759 	if (ppc_md.init_IRQ)
760 		ppc_md.init_IRQ();
761 }
762 
763 #if defined(CONFIG_BOOKE) || defined(CONFIG_40x)
764 void   *critirq_ctx[NR_CPUS] __read_mostly;
765 void    *dbgirq_ctx[NR_CPUS] __read_mostly;
766 void *mcheckirq_ctx[NR_CPUS] __read_mostly;
767 #endif
768 
769 void *softirq_ctx[NR_CPUS] __read_mostly;
770 void *hardirq_ctx[NR_CPUS] __read_mostly;
771 
772 void do_softirq_own_stack(void)
773 {
774 	call_do_softirq(softirq_ctx[smp_processor_id()]);
775 }
776 
777 irq_hw_number_t virq_to_hw(unsigned int virq)
778 {
779 	struct irq_data *irq_data = irq_get_irq_data(virq);
780 	return WARN_ON(!irq_data) ? 0 : irq_data->hwirq;
781 }
782 EXPORT_SYMBOL_GPL(virq_to_hw);
783 
784 #ifdef CONFIG_SMP
785 int irq_choose_cpu(const struct cpumask *mask)
786 {
787 	int cpuid;
788 
789 	if (cpumask_equal(mask, cpu_online_mask)) {
790 		static int irq_rover;
791 		static DEFINE_RAW_SPINLOCK(irq_rover_lock);
792 		unsigned long flags;
793 
794 		/* Round-robin distribution... */
795 do_round_robin:
796 		raw_spin_lock_irqsave(&irq_rover_lock, flags);
797 
798 		irq_rover = cpumask_next(irq_rover, cpu_online_mask);
799 		if (irq_rover >= nr_cpu_ids)
800 			irq_rover = cpumask_first(cpu_online_mask);
801 
802 		cpuid = irq_rover;
803 
804 		raw_spin_unlock_irqrestore(&irq_rover_lock, flags);
805 	} else {
806 		cpuid = cpumask_first_and(mask, cpu_online_mask);
807 		if (cpuid >= nr_cpu_ids)
808 			goto do_round_robin;
809 	}
810 
811 	return get_hard_smp_processor_id(cpuid);
812 }
813 #else
814 int irq_choose_cpu(const struct cpumask *mask)
815 {
816 	return hard_smp_processor_id();
817 }
818 #endif
819 
820 #ifdef CONFIG_PPC64
821 static int __init setup_noirqdistrib(char *str)
822 {
823 	distribute_irqs = 0;
824 	return 1;
825 }
826 
827 __setup("noirqdistrib", setup_noirqdistrib);
828 #endif /* CONFIG_PPC64 */
829