1 #ifndef _ASM_X86_SPECIAL_INSNS_H 2 #define _ASM_X86_SPECIAL_INSNS_H 3 4 5 #ifdef __KERNEL__ 6 7 #include <asm/nops.h> 8 9 static inline void native_clts(void) 10 { 11 asm volatile("clts"); 12 } 13 14 /* 15 * Volatile isn't enough to prevent the compiler from reordering the 16 * read/write functions for the control registers and messing everything up. 17 * A memory clobber would solve the problem, but would prevent reordering of 18 * all loads stores around it, which can hurt performance. Solution is to 19 * use a variable and mimic reads and writes to it to enforce serialization 20 */ 21 extern unsigned long __force_order; 22 23 static inline unsigned long native_read_cr0(void) 24 { 25 unsigned long val; 26 asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order)); 27 return val; 28 } 29 30 static inline void native_write_cr0(unsigned long val) 31 { 32 asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order)); 33 } 34 35 static inline unsigned long native_read_cr2(void) 36 { 37 unsigned long val; 38 asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order)); 39 return val; 40 } 41 42 static inline void native_write_cr2(unsigned long val) 43 { 44 asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order)); 45 } 46 47 static inline unsigned long native_read_cr3(void) 48 { 49 unsigned long val; 50 asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order)); 51 return val; 52 } 53 54 static inline void native_write_cr3(unsigned long val) 55 { 56 asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order)); 57 } 58 59 static inline unsigned long native_read_cr4(void) 60 { 61 unsigned long val; 62 asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order)); 63 return val; 64 } 65 66 static inline unsigned long native_read_cr4_safe(void) 67 { 68 unsigned long val; 69 /* This could fault if %cr4 does not exist. In x86_64, a cr4 always 70 * exists, so it will never fail. */ 71 #ifdef CONFIG_X86_32 72 asm volatile("1: mov %%cr4, %0\n" 73 "2:\n" 74 _ASM_EXTABLE(1b, 2b) 75 : "=r" (val), "=m" (__force_order) : "0" (0)); 76 #else 77 val = native_read_cr4(); 78 #endif 79 return val; 80 } 81 82 static inline void native_write_cr4(unsigned long val) 83 { 84 asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order)); 85 } 86 87 #ifdef CONFIG_X86_64 88 static inline unsigned long native_read_cr8(void) 89 { 90 unsigned long cr8; 91 asm volatile("movq %%cr8,%0" : "=r" (cr8)); 92 return cr8; 93 } 94 95 static inline void native_write_cr8(unsigned long val) 96 { 97 asm volatile("movq %0,%%cr8" :: "r" (val) : "memory"); 98 } 99 #endif 100 101 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS 102 static inline u32 __read_pkru(void) 103 { 104 u32 ecx = 0; 105 u32 edx, pkru; 106 107 /* 108 * "rdpkru" instruction. Places PKRU contents in to EAX, 109 * clears EDX and requires that ecx=0. 110 */ 111 asm volatile(".byte 0x0f,0x01,0xee\n\t" 112 : "=a" (pkru), "=d" (edx) 113 : "c" (ecx)); 114 return pkru; 115 } 116 117 static inline void __write_pkru(u32 pkru) 118 { 119 u32 ecx = 0, edx = 0; 120 121 /* 122 * "wrpkru" instruction. Loads contents in EAX to PKRU, 123 * requires that ecx = edx = 0. 124 */ 125 asm volatile(".byte 0x0f,0x01,0xef\n\t" 126 : : "a" (pkru), "c"(ecx), "d"(edx)); 127 } 128 #else 129 static inline u32 __read_pkru(void) 130 { 131 return 0; 132 } 133 134 static inline void __write_pkru(u32 pkru) 135 { 136 } 137 #endif 138 139 static inline void native_wbinvd(void) 140 { 141 asm volatile("wbinvd": : :"memory"); 142 } 143 144 extern asmlinkage void native_load_gs_index(unsigned); 145 146 #ifdef CONFIG_PARAVIRT 147 #include <asm/paravirt.h> 148 #else 149 150 static inline unsigned long read_cr0(void) 151 { 152 return native_read_cr0(); 153 } 154 155 static inline void write_cr0(unsigned long x) 156 { 157 native_write_cr0(x); 158 } 159 160 static inline unsigned long read_cr2(void) 161 { 162 return native_read_cr2(); 163 } 164 165 static inline void write_cr2(unsigned long x) 166 { 167 native_write_cr2(x); 168 } 169 170 static inline unsigned long read_cr3(void) 171 { 172 return native_read_cr3(); 173 } 174 175 static inline void write_cr3(unsigned long x) 176 { 177 native_write_cr3(x); 178 } 179 180 static inline unsigned long __read_cr4(void) 181 { 182 return native_read_cr4(); 183 } 184 185 static inline unsigned long __read_cr4_safe(void) 186 { 187 return native_read_cr4_safe(); 188 } 189 190 static inline void __write_cr4(unsigned long x) 191 { 192 native_write_cr4(x); 193 } 194 195 static inline void wbinvd(void) 196 { 197 native_wbinvd(); 198 } 199 200 #ifdef CONFIG_X86_64 201 202 static inline unsigned long read_cr8(void) 203 { 204 return native_read_cr8(); 205 } 206 207 static inline void write_cr8(unsigned long x) 208 { 209 native_write_cr8(x); 210 } 211 212 static inline void load_gs_index(unsigned selector) 213 { 214 native_load_gs_index(selector); 215 } 216 217 #endif 218 219 /* Clear the 'TS' bit */ 220 static inline void clts(void) 221 { 222 native_clts(); 223 } 224 225 #endif/* CONFIG_PARAVIRT */ 226 227 #define stts() write_cr0(read_cr0() | X86_CR0_TS) 228 229 static inline void clflush(volatile void *__p) 230 { 231 asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p)); 232 } 233 234 static inline void clflushopt(volatile void *__p) 235 { 236 alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0", 237 ".byte 0x66; clflush %P0", 238 X86_FEATURE_CLFLUSHOPT, 239 "+m" (*(volatile char __force *)__p)); 240 } 241 242 static inline void clwb(volatile void *__p) 243 { 244 volatile struct { char x[64]; } *p = __p; 245 246 asm volatile(ALTERNATIVE_2( 247 ".byte " __stringify(NOP_DS_PREFIX) "; clflush (%[pax])", 248 ".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */ 249 X86_FEATURE_CLFLUSHOPT, 250 ".byte 0x66, 0x0f, 0xae, 0x30", /* clwb (%%rax) */ 251 X86_FEATURE_CLWB) 252 : [p] "+m" (*p) 253 : [pax] "a" (p)); 254 } 255 256 #define nop() asm volatile ("nop") 257 258 259 #endif /* __KERNEL__ */ 260 261 #endif /* _ASM_X86_SPECIAL_INSNS_H */ 262