1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_IRQ_STACK_H 3 #define _ASM_X86_IRQ_STACK_H 4 5 #include <linux/ptrace.h> 6 7 #include <asm/processor.h> 8 9 #ifdef CONFIG_X86_64 10 static __always_inline bool irqstack_active(void) 11 { 12 return __this_cpu_read(irq_count) != -1; 13 } 14 15 void asm_call_on_stack(void *sp, void (*func)(void), void *arg); 16 void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs), 17 struct pt_regs *regs); 18 void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc), 19 struct irq_desc *desc); 20 21 static __always_inline void __run_on_irqstack(void (*func)(void)) 22 { 23 void *tos = __this_cpu_read(hardirq_stack_ptr); 24 25 __this_cpu_add(irq_count, 1); 26 asm_call_on_stack(tos - 8, func, NULL); 27 __this_cpu_sub(irq_count, 1); 28 } 29 30 static __always_inline void 31 __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), 32 struct pt_regs *regs) 33 { 34 void *tos = __this_cpu_read(hardirq_stack_ptr); 35 36 __this_cpu_add(irq_count, 1); 37 asm_call_sysvec_on_stack(tos - 8, func, regs); 38 __this_cpu_sub(irq_count, 1); 39 } 40 41 static __always_inline void 42 __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), 43 struct irq_desc *desc) 44 { 45 void *tos = __this_cpu_read(hardirq_stack_ptr); 46 47 __this_cpu_add(irq_count, 1); 48 asm_call_irq_on_stack(tos - 8, func, desc); 49 __this_cpu_sub(irq_count, 1); 50 } 51 52 #else /* CONFIG_X86_64 */ 53 static inline bool irqstack_active(void) { return false; } 54 static inline void __run_on_irqstack(void (*func)(void)) { } 55 static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), 56 struct pt_regs *regs) { } 57 static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), 58 struct irq_desc *desc) { } 59 #endif /* !CONFIG_X86_64 */ 60 61 static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs) 62 { 63 if (IS_ENABLED(CONFIG_X86_32)) 64 return false; 65 if (!regs) 66 return !irqstack_active(); 67 return !user_mode(regs) && !irqstack_active(); 68 } 69 70 71 static __always_inline void run_on_irqstack_cond(void (*func)(void), 72 struct pt_regs *regs) 73 { 74 lockdep_assert_irqs_disabled(); 75 76 if (irq_needs_irq_stack(regs)) 77 __run_on_irqstack(func); 78 else 79 func(); 80 } 81 82 static __always_inline void 83 run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs), 84 struct pt_regs *regs) 85 { 86 lockdep_assert_irqs_disabled(); 87 88 if (irq_needs_irq_stack(regs)) 89 __run_sysvec_on_irqstack(func, regs); 90 else 91 func(regs); 92 } 93 94 static __always_inline void 95 run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc, 96 struct pt_regs *regs) 97 { 98 lockdep_assert_irqs_disabled(); 99 100 if (irq_needs_irq_stack(regs)) 101 __run_irq_on_irqstack(func, desc); 102 else 103 func(desc); 104 } 105 106 #endif 107