1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _X86_IRQFLAGS_H_ 3 #define _X86_IRQFLAGS_H_ 4 5 #include <asm/processor-flags.h> 6 7 #ifndef __ASSEMBLY__ 8 9 /* Provide __cpuidle; we can't safely include <linux/cpu.h> */ 10 #define __cpuidle __attribute__((__section__(".cpuidle.text"))) 11 12 /* 13 * Interrupt control: 14 */ 15 16 /* Declaration required for gcc < 4.9 to prevent -Werror=missing-prototypes */ 17 extern inline unsigned long native_save_fl(void); 18 extern inline unsigned long native_save_fl(void) 19 { 20 unsigned long flags; 21 22 /* 23 * "=rm" is safe here, because "pop" adjusts the stack before 24 * it evaluates its effective address -- this is part of the 25 * documented behavior of the "pop" instruction. 26 */ 27 asm volatile("# __raw_save_flags\n\t" 28 "pushf ; pop %0" 29 : "=rm" (flags) 30 : /* no input */ 31 : "memory"); 32 33 return flags; 34 } 35 36 static inline void native_restore_fl(unsigned long flags) 37 { 38 asm volatile("push %0 ; popf" 39 : /* no output */ 40 :"g" (flags) 41 :"memory", "cc"); 42 } 43 44 static inline void native_irq_disable(void) 45 { 46 asm volatile("cli": : :"memory"); 47 } 48 49 static inline void native_irq_enable(void) 50 { 51 asm volatile("sti": : :"memory"); 52 } 53 54 static inline __cpuidle void native_safe_halt(void) 55 { 56 asm volatile("sti; hlt": : :"memory"); 57 } 58 59 static inline __cpuidle void native_halt(void) 60 { 61 asm volatile("hlt": : :"memory"); 62 } 63 64 #endif 65 66 #ifdef CONFIG_PARAVIRT 67 #include <asm/paravirt.h> 68 #else 69 #ifndef __ASSEMBLY__ 70 #include <linux/types.h> 71 72 static inline notrace unsigned long arch_local_save_flags(void) 73 { 74 return native_save_fl(); 75 } 76 77 static inline notrace void arch_local_irq_restore(unsigned long flags) 78 { 79 native_restore_fl(flags); 80 } 81 82 static inline notrace void arch_local_irq_disable(void) 83 { 84 native_irq_disable(); 85 } 86 87 static inline notrace void arch_local_irq_enable(void) 88 { 89 native_irq_enable(); 90 } 91 92 /* 93 * Used in the idle loop; sti takes one instruction cycle 94 * to complete: 95 */ 96 static inline __cpuidle void arch_safe_halt(void) 97 { 98 native_safe_halt(); 99 } 100 101 /* 102 * Used when interrupts are already enabled or to 103 * shutdown the processor: 104 */ 105 static inline __cpuidle void halt(void) 106 { 107 native_halt(); 108 } 109 110 /* 111 * For spinlocks, etc: 112 */ 113 static inline notrace unsigned long arch_local_irq_save(void) 114 { 115 unsigned long flags = arch_local_save_flags(); 116 arch_local_irq_disable(); 117 return flags; 118 } 119 #else 120 121 #define ENABLE_INTERRUPTS(x) sti 122 #define DISABLE_INTERRUPTS(x) cli 123 124 #ifdef CONFIG_X86_64 125 #define SWAPGS swapgs 126 /* 127 * Currently paravirt can't handle swapgs nicely when we 128 * don't have a stack we can rely on (such as a user space 129 * stack). So we either find a way around these or just fault 130 * and emulate if a guest tries to call swapgs directly. 131 * 132 * Either way, this is a good way to document that we don't 133 * have a reliable stack. x86_64 only. 134 */ 135 #define SWAPGS_UNSAFE_STACK swapgs 136 137 #define PARAVIRT_ADJUST_EXCEPTION_FRAME /* */ 138 139 #define INTERRUPT_RETURN jmp native_iret 140 #define USERGS_SYSRET64 \ 141 swapgs; \ 142 sysretq; 143 #define USERGS_SYSRET32 \ 144 swapgs; \ 145 sysretl 146 147 #ifdef CONFIG_DEBUG_ENTRY 148 #define SAVE_FLAGS(x) pushfq; popq %rax 149 #endif 150 #else 151 #define INTERRUPT_RETURN iret 152 #define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit 153 #define GET_CR0_INTO_EAX movl %cr0, %eax 154 #endif 155 156 157 #endif /* __ASSEMBLY__ */ 158 #endif /* CONFIG_PARAVIRT */ 159 160 #ifndef __ASSEMBLY__ 161 static inline int arch_irqs_disabled_flags(unsigned long flags) 162 { 163 return !(flags & X86_EFLAGS_IF); 164 } 165 166 static inline int arch_irqs_disabled(void) 167 { 168 unsigned long flags = arch_local_save_flags(); 169 170 return arch_irqs_disabled_flags(flags); 171 } 172 #endif /* !__ASSEMBLY__ */ 173 174 #ifdef __ASSEMBLY__ 175 #ifdef CONFIG_TRACE_IRQFLAGS 176 # define TRACE_IRQS_ON call trace_hardirqs_on_thunk; 177 # define TRACE_IRQS_OFF call trace_hardirqs_off_thunk; 178 #else 179 # define TRACE_IRQS_ON 180 # define TRACE_IRQS_OFF 181 #endif 182 #ifdef CONFIG_DEBUG_LOCK_ALLOC 183 # ifdef CONFIG_X86_64 184 # define LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk 185 # define LOCKDEP_SYS_EXIT_IRQ \ 186 TRACE_IRQS_ON; \ 187 sti; \ 188 call lockdep_sys_exit_thunk; \ 189 cli; \ 190 TRACE_IRQS_OFF; 191 # else 192 # define LOCKDEP_SYS_EXIT \ 193 pushl %eax; \ 194 pushl %ecx; \ 195 pushl %edx; \ 196 call lockdep_sys_exit; \ 197 popl %edx; \ 198 popl %ecx; \ 199 popl %eax; 200 # define LOCKDEP_SYS_EXIT_IRQ 201 # endif 202 #else 203 # define LOCKDEP_SYS_EXIT 204 # define LOCKDEP_SYS_EXIT_IRQ 205 #endif 206 #endif /* __ASSEMBLY__ */ 207 208 #endif 209