xref: /openbmc/linux/arch/x86/include/asm/irq_stack.h (revision 5396ce9a)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_IRQ_STACK_H
3 #define _ASM_X86_IRQ_STACK_H
4 
5 #include <linux/ptrace.h>
6 #include <linux/objtool.h>
7 
8 #include <asm/processor.h>
9 
10 #ifdef CONFIG_X86_64
11 
12 /*
13  * Macro to inline switching to an interrupt stack and invoking function
14  * calls from there. The following rules apply:
15  *
16  * - Ordering:
17  *
18  *   1. Write the stack pointer into the top most place of the irq
19  *	stack. This ensures that the various unwinders can link back to the
20  *	original stack.
21  *
22  *   2. Switch the stack pointer to the top of the irq stack.
23  *
24  *   3. Invoke whatever needs to be done (@asm_call argument)
25  *
26  *   4. Pop the original stack pointer from the top of the irq stack
27  *	which brings it back to the original stack where it left off.
28  *
29  * - Function invocation:
30  *
31  *   To allow flexible usage of the macro, the actual function code including
32  *   the store of the arguments in the call ABI registers is handed in via
33  *   the @asm_call argument.
34  *
35  * - Local variables:
36  *
37  *   @tos:
38  *	The @tos variable holds a pointer to the top of the irq stack and
39  *	_must_ be allocated in a non-callee saved register as this is a
40  *	restriction coming from objtool.
41  *
42  *	Note, that (tos) is both in input and output constraints to ensure
43  *	that the compiler does not assume that R11 is left untouched in
44  *	case this macro is used in some place where the per cpu interrupt
45  *	stack pointer is used again afterwards
46  *
47  * - Function arguments:
48  *	The function argument(s), if any, have to be defined in register
49  *	variables at the place where this is invoked. Storing the
50  *	argument(s) in the proper register(s) is part of the @asm_call
51  *
52  * - Constraints:
53  *
54  *   The constraints have to be done very carefully because the compiler
55  *   does not know about the assembly call.
56  *
57  *   output:
58  *     As documented already above the @tos variable is required to be in
59  *     the output constraints to make the compiler aware that R11 cannot be
60  *     reused after the asm() statement.
61  *
62  *     For builds with CONFIG_UNWINDER_FRAME_POINTER, ASM_CALL_CONSTRAINT is
63  *     required as well as this prevents certain creative GCC variants from
64  *     misplacing the ASM code.
65  *
66  *  input:
67  *    - func:
68  *	  Immediate, which tells the compiler that the function is referenced.
69  *
70  *    - tos:
71  *	  Register. The actual register is defined by the variable declaration.
72  *
73  *    - function arguments:
74  *	  The constraints are handed in via the 'argconstr' argument list. They
75  *	  describe the register arguments which are used in @asm_call.
76  *
77  *  clobbers:
78  *     Function calls can clobber anything except the callee-saved
79  *     registers. Tell the compiler.
80  */
81 #define call_on_stack(stack, func, asm_call, argconstr...)		\
82 {									\
83 	register void *tos asm("r11");					\
84 									\
85 	tos = ((void *)(stack));					\
86 									\
87 	asm_inline volatile(						\
88 	"movq	%%rsp, (%[tos])				\n"		\
89 	"movq	%[tos], %%rsp				\n"		\
90 									\
91 	asm_call							\
92 									\
93 	"popq	%%rsp					\n"		\
94 									\
95 	: "+r" (tos), ASM_CALL_CONSTRAINT				\
96 	: [__func] "i" (func), [tos] "r" (tos) argconstr		\
97 	: "cc", "rax", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10",	\
98 	  "memory"							\
99 	);								\
100 }
101 
102 #define ASM_CALL_ARG0							\
103 	"call %c[__func]				\n"		\
104 	ASM_REACHABLE
105 
106 #define ASM_CALL_ARG1							\
107 	"movq	%[arg1], %%rdi				\n"		\
108 	ASM_CALL_ARG0
109 
110 #define ASM_CALL_ARG2							\
111 	"movq	%[arg2], %%rsi				\n"		\
112 	ASM_CALL_ARG1
113 
114 #define ASM_CALL_ARG3							\
115 	"movq	%[arg3], %%rdx				\n"		\
116 	ASM_CALL_ARG2
117 
118 #define call_on_irqstack(func, asm_call, argconstr...)			\
119 	call_on_stack(__this_cpu_read(pcpu_hot.hardirq_stack_ptr),	\
120 		      func, asm_call, argconstr)
121 
122 /* Macros to assert type correctness for run_*_on_irqstack macros */
123 #define assert_function_type(func, proto)				\
124 	static_assert(__builtin_types_compatible_p(typeof(&func), proto))
125 
126 #define assert_arg_type(arg, proto)					\
127 	static_assert(__builtin_types_compatible_p(typeof(arg), proto))
128 
129 /*
130  * Macro to invoke system vector and device interrupt C handlers.
131  */
132 #define call_on_irqstack_cond(func, regs, asm_call, constr, c_args...)	\
133 {									\
134 	/*								\
135 	 * User mode entry and interrupt on the irq stack do not	\
136 	 * switch stacks. If from user mode the task stack is empty.	\
137 	 */								\
138 	if (user_mode(regs) || __this_cpu_read(pcpu_hot.hardirq_stack_inuse)) { \
139 		irq_enter_rcu();					\
140 		func(c_args);						\
141 		irq_exit_rcu();						\
142 	} else {							\
143 		/*							\
144 		 * Mark the irq stack inuse _before_ and unmark _after_	\
145 		 * switching stacks. Interrupts are disabled in both	\
146 		 * places. Invoke the stack switch macro with the call	\
147 		 * sequence which matches the above direct invocation.	\
148 		 */							\
149 		__this_cpu_write(pcpu_hot.hardirq_stack_inuse, true);	\
150 		call_on_irqstack(func, asm_call, constr);		\
151 		__this_cpu_write(pcpu_hot.hardirq_stack_inuse, false);	\
152 	}								\
153 }
154 
155 /*
156  * Function call sequence for __call_on_irqstack() for system vectors.
157  *
158  * Note that irq_enter_rcu() and irq_exit_rcu() do not use the input
159  * mechanism because these functions are global and cannot be optimized out
160  * when compiling a particular source file which uses one of these macros.
161  *
162  * The argument (regs) does not need to be pushed or stashed in a callee
163  * saved register to be safe vs. the irq_enter_rcu() call because the
164  * clobbers already prevent the compiler from storing it in a callee
165  * clobbered register. As the compiler has to preserve @regs for the final
166  * call to idtentry_exit() anyway, it's likely that it does not cause extra
167  * effort for this asm magic.
168  */
169 #define ASM_CALL_SYSVEC							\
170 	"call irq_enter_rcu				\n"		\
171 	ASM_CALL_ARG1							\
172 	"call irq_exit_rcu				\n"
173 
174 #define SYSVEC_CONSTRAINTS	, [arg1] "r" (regs)
175 
176 #define run_sysvec_on_irqstack_cond(func, regs)				\
177 {									\
178 	assert_function_type(func, void (*)(struct pt_regs *));		\
179 	assert_arg_type(regs, struct pt_regs *);			\
180 									\
181 	call_on_irqstack_cond(func, regs, ASM_CALL_SYSVEC,		\
182 			      SYSVEC_CONSTRAINTS, regs);		\
183 }
184 
185 /*
186  * As in ASM_CALL_SYSVEC above the clobbers force the compiler to store
187  * @regs and @vector in callee saved registers.
188  */
189 #define ASM_CALL_IRQ							\
190 	"call irq_enter_rcu				\n"		\
191 	ASM_CALL_ARG2							\
192 	"call irq_exit_rcu				\n"
193 
194 #define IRQ_CONSTRAINTS	, [arg1] "r" (regs), [arg2] "r" ((unsigned long)vector)
195 
196 #define run_irq_on_irqstack_cond(func, regs, vector)			\
197 {									\
198 	assert_function_type(func, void (*)(struct pt_regs *, u32));	\
199 	assert_arg_type(regs, struct pt_regs *);			\
200 	assert_arg_type(vector, u32);					\
201 									\
202 	call_on_irqstack_cond(func, regs, ASM_CALL_IRQ,			\
203 			      IRQ_CONSTRAINTS, regs, vector);		\
204 }
205 
206 #ifdef CONFIG_SOFTIRQ_ON_OWN_STACK
207 /*
208  * Macro to invoke __do_softirq on the irq stack. This is only called from
209  * task context when bottom halves are about to be reenabled and soft
210  * interrupts are pending to be processed. The interrupt stack cannot be in
211  * use here.
212  */
213 #define do_softirq_own_stack()						\
214 {									\
215 	__this_cpu_write(pcpu_hot.hardirq_stack_inuse, true);		\
216 	call_on_irqstack(__do_softirq, ASM_CALL_ARG0);			\
217 	__this_cpu_write(pcpu_hot.hardirq_stack_inuse, false);		\
218 }
219 
220 #endif
221 
222 #else /* CONFIG_X86_64 */
223 /* System vector handlers always run on the stack they interrupted. */
224 #define run_sysvec_on_irqstack_cond(func, regs)				\
225 {									\
226 	irq_enter_rcu();						\
227 	func(regs);							\
228 	irq_exit_rcu();							\
229 }
230 
231 /* Switches to the irq stack within func() */
232 #define run_irq_on_irqstack_cond(func, regs, vector)			\
233 {									\
234 	irq_enter_rcu();						\
235 	func(regs, vector);						\
236 	irq_exit_rcu();							\
237 }
238 
239 #endif /* !CONFIG_X86_64 */
240 
241 #endif
242