1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _X86_IRQFLAGS_H_ 3 #define _X86_IRQFLAGS_H_ 4 5 #include <asm/processor-flags.h> 6 7 #ifndef __ASSEMBLY__ 8 9 /* Provide __cpuidle; we can't safely include <linux/cpu.h> */ 10 #define __cpuidle __attribute__((__section__(".cpuidle.text"))) 11 12 /* 13 * Interrupt control: 14 */ 15 16 static inline unsigned long native_save_fl(void) 17 { 18 unsigned long flags; 19 20 /* 21 * "=rm" is safe here, because "pop" adjusts the stack before 22 * it evaluates its effective address -- this is part of the 23 * documented behavior of the "pop" instruction. 24 */ 25 asm volatile("# __raw_save_flags\n\t" 26 "pushf ; pop %0" 27 : "=rm" (flags) 28 : /* no input */ 29 : "memory"); 30 31 return flags; 32 } 33 34 static inline void native_restore_fl(unsigned long flags) 35 { 36 asm volatile("push %0 ; popf" 37 : /* no output */ 38 :"g" (flags) 39 :"memory", "cc"); 40 } 41 42 static inline void native_irq_disable(void) 43 { 44 asm volatile("cli": : :"memory"); 45 } 46 47 static inline void native_irq_enable(void) 48 { 49 asm volatile("sti": : :"memory"); 50 } 51 52 static inline __cpuidle void native_safe_halt(void) 53 { 54 asm volatile("sti; hlt": : :"memory"); 55 } 56 57 static inline __cpuidle void native_halt(void) 58 { 59 asm volatile("hlt": : :"memory"); 60 } 61 62 #endif 63 64 #ifdef CONFIG_PARAVIRT 65 #include <asm/paravirt.h> 66 #else 67 #ifndef __ASSEMBLY__ 68 #include <linux/types.h> 69 70 static inline notrace unsigned long arch_local_save_flags(void) 71 { 72 return native_save_fl(); 73 } 74 75 static inline notrace void arch_local_irq_restore(unsigned long flags) 76 { 77 native_restore_fl(flags); 78 } 79 80 static inline notrace void arch_local_irq_disable(void) 81 { 82 native_irq_disable(); 83 } 84 85 static inline notrace void arch_local_irq_enable(void) 86 { 87 native_irq_enable(); 88 } 89 90 /* 91 * Used in the idle loop; sti takes one instruction cycle 92 * to complete: 93 */ 94 static inline __cpuidle void arch_safe_halt(void) 95 { 96 native_safe_halt(); 97 } 98 99 /* 100 * Used when interrupts are already enabled or to 101 * shutdown the processor: 102 */ 103 static inline __cpuidle void halt(void) 104 { 105 native_halt(); 106 } 107 108 /* 109 * For spinlocks, etc: 110 */ 111 static inline notrace unsigned long arch_local_irq_save(void) 112 { 113 unsigned long flags = arch_local_save_flags(); 114 arch_local_irq_disable(); 115 return flags; 116 } 117 #else 118 119 #define ENABLE_INTERRUPTS(x) sti 120 #define DISABLE_INTERRUPTS(x) cli 121 122 #ifdef CONFIG_X86_64 123 #define SWAPGS swapgs 124 /* 125 * Currently paravirt can't handle swapgs nicely when we 126 * don't have a stack we can rely on (such as a user space 127 * stack). So we either find a way around these or just fault 128 * and emulate if a guest tries to call swapgs directly. 129 * 130 * Either way, this is a good way to document that we don't 131 * have a reliable stack. x86_64 only. 132 */ 133 #define SWAPGS_UNSAFE_STACK swapgs 134 135 #define PARAVIRT_ADJUST_EXCEPTION_FRAME /* */ 136 137 #define INTERRUPT_RETURN jmp native_iret 138 #define USERGS_SYSRET64 \ 139 swapgs; \ 140 sysretq; 141 #define USERGS_SYSRET32 \ 142 swapgs; \ 143 sysretl 144 145 #else 146 #define INTERRUPT_RETURN iret 147 #define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit 148 #define GET_CR0_INTO_EAX movl %cr0, %eax 149 #endif 150 151 152 #endif /* __ASSEMBLY__ */ 153 #endif /* CONFIG_PARAVIRT */ 154 155 #ifndef __ASSEMBLY__ 156 static inline int arch_irqs_disabled_flags(unsigned long flags) 157 { 158 return !(flags & X86_EFLAGS_IF); 159 } 160 161 static inline int arch_irqs_disabled(void) 162 { 163 unsigned long flags = arch_local_save_flags(); 164 165 return arch_irqs_disabled_flags(flags); 166 } 167 #endif /* !__ASSEMBLY__ */ 168 169 #ifdef __ASSEMBLY__ 170 #ifdef CONFIG_TRACE_IRQFLAGS 171 # define TRACE_IRQS_ON call trace_hardirqs_on_thunk; 172 # define TRACE_IRQS_OFF call trace_hardirqs_off_thunk; 173 #else 174 # define TRACE_IRQS_ON 175 # define TRACE_IRQS_OFF 176 #endif 177 #ifdef CONFIG_DEBUG_LOCK_ALLOC 178 # ifdef CONFIG_X86_64 179 # define LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk 180 # define LOCKDEP_SYS_EXIT_IRQ \ 181 TRACE_IRQS_ON; \ 182 sti; \ 183 call lockdep_sys_exit_thunk; \ 184 cli; \ 185 TRACE_IRQS_OFF; 186 # else 187 # define LOCKDEP_SYS_EXIT \ 188 pushl %eax; \ 189 pushl %ecx; \ 190 pushl %edx; \ 191 call lockdep_sys_exit; \ 192 popl %edx; \ 193 popl %ecx; \ 194 popl %eax; 195 # define LOCKDEP_SYS_EXIT_IRQ 196 # endif 197 #else 198 # define LOCKDEP_SYS_EXIT 199 # define LOCKDEP_SYS_EXIT_IRQ 200 #endif 201 #endif /* __ASSEMBLY__ */ 202 203 #endif 204