1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_SPECIAL_INSNS_H 3 #define _ASM_X86_SPECIAL_INSNS_H 4 5 6 #ifdef __KERNEL__ 7 8 #include <asm/nops.h> 9 10 /* 11 * Volatile isn't enough to prevent the compiler from reordering the 12 * read/write functions for the control registers and messing everything up. 13 * A memory clobber would solve the problem, but would prevent reordering of 14 * all loads stores around it, which can hurt performance. Solution is to 15 * use a variable and mimic reads and writes to it to enforce serialization 16 */ 17 extern unsigned long __force_order; 18 19 static inline unsigned long native_read_cr0(void) 20 { 21 unsigned long val; 22 asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order)); 23 return val; 24 } 25 26 static inline void native_write_cr0(unsigned long val) 27 { 28 asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order)); 29 } 30 31 static inline unsigned long native_read_cr2(void) 32 { 33 unsigned long val; 34 asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order)); 35 return val; 36 } 37 38 static inline void native_write_cr2(unsigned long val) 39 { 40 asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order)); 41 } 42 43 static inline unsigned long __native_read_cr3(void) 44 { 45 unsigned long val; 46 asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order)); 47 return val; 48 } 49 50 static inline void native_write_cr3(unsigned long val) 51 { 52 asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order)); 53 } 54 55 static inline unsigned long native_read_cr4(void) 56 { 57 unsigned long val; 58 #ifdef CONFIG_X86_32 59 /* 60 * This could fault if CR4 does not exist. Non-existent CR4 61 * is functionally equivalent to CR4 == 0. Keep it simple and pretend 62 * that CR4 == 0 on CPUs that don't have CR4. 63 */ 64 asm volatile("1: mov %%cr4, %0\n" 65 "2:\n" 66 _ASM_EXTABLE(1b, 2b) 67 : "=r" (val), "=m" (__force_order) : "0" (0)); 68 #else 69 /* CR4 always exists on x86_64. */ 70 asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order)); 71 #endif 72 return val; 73 } 74 75 static inline void native_write_cr4(unsigned long val) 76 { 77 asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order)); 78 } 79 80 #ifdef CONFIG_X86_64 81 static inline unsigned long native_read_cr8(void) 82 { 83 unsigned long cr8; 84 asm volatile("movq %%cr8,%0" : "=r" (cr8)); 85 return cr8; 86 } 87 88 static inline void native_write_cr8(unsigned long val) 89 { 90 asm volatile("movq %0,%%cr8" :: "r" (val) : "memory"); 91 } 92 #endif 93 94 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS 95 static inline u32 rdpkru(void) 96 { 97 u32 ecx = 0; 98 u32 edx, pkru; 99 100 /* 101 * "rdpkru" instruction. Places PKRU contents in to EAX, 102 * clears EDX and requires that ecx=0. 103 */ 104 asm volatile(".byte 0x0f,0x01,0xee\n\t" 105 : "=a" (pkru), "=d" (edx) 106 : "c" (ecx)); 107 return pkru; 108 } 109 110 static inline void wrpkru(u32 pkru) 111 { 112 u32 ecx = 0, edx = 0; 113 114 /* 115 * "wrpkru" instruction. Loads contents in EAX to PKRU, 116 * requires that ecx = edx = 0. 117 */ 118 asm volatile(".byte 0x0f,0x01,0xef\n\t" 119 : : "a" (pkru), "c"(ecx), "d"(edx)); 120 } 121 122 static inline void __write_pkru(u32 pkru) 123 { 124 /* 125 * WRPKRU is relatively expensive compared to RDPKRU. 126 * Avoid WRPKRU when it would not change the value. 127 */ 128 if (pkru == rdpkru()) 129 return; 130 131 wrpkru(pkru); 132 } 133 134 #else 135 static inline u32 rdpkru(void) 136 { 137 return 0; 138 } 139 140 static inline void __write_pkru(u32 pkru) 141 { 142 } 143 #endif 144 145 static inline void native_wbinvd(void) 146 { 147 asm volatile("wbinvd": : :"memory"); 148 } 149 150 extern asmlinkage void native_load_gs_index(unsigned); 151 152 static inline unsigned long __read_cr4(void) 153 { 154 return native_read_cr4(); 155 } 156 157 #ifdef CONFIG_PARAVIRT_XXL 158 #include <asm/paravirt.h> 159 #else 160 161 static inline unsigned long read_cr0(void) 162 { 163 return native_read_cr0(); 164 } 165 166 static inline void write_cr0(unsigned long x) 167 { 168 native_write_cr0(x); 169 } 170 171 static inline unsigned long read_cr2(void) 172 { 173 return native_read_cr2(); 174 } 175 176 static inline void write_cr2(unsigned long x) 177 { 178 native_write_cr2(x); 179 } 180 181 /* 182 * Careful! CR3 contains more than just an address. You probably want 183 * read_cr3_pa() instead. 184 */ 185 static inline unsigned long __read_cr3(void) 186 { 187 return __native_read_cr3(); 188 } 189 190 static inline void write_cr3(unsigned long x) 191 { 192 native_write_cr3(x); 193 } 194 195 static inline void __write_cr4(unsigned long x) 196 { 197 native_write_cr4(x); 198 } 199 200 static inline void wbinvd(void) 201 { 202 native_wbinvd(); 203 } 204 205 #ifdef CONFIG_X86_64 206 207 static inline unsigned long read_cr8(void) 208 { 209 return native_read_cr8(); 210 } 211 212 static inline void write_cr8(unsigned long x) 213 { 214 native_write_cr8(x); 215 } 216 217 static inline void load_gs_index(unsigned selector) 218 { 219 native_load_gs_index(selector); 220 } 221 222 #endif 223 224 #endif /* CONFIG_PARAVIRT_XXL */ 225 226 static inline void clflush(volatile void *__p) 227 { 228 asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p)); 229 } 230 231 static inline void clflushopt(volatile void *__p) 232 { 233 alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0", 234 ".byte 0x66; clflush %P0", 235 X86_FEATURE_CLFLUSHOPT, 236 "+m" (*(volatile char __force *)__p)); 237 } 238 239 static inline void clwb(volatile void *__p) 240 { 241 volatile struct { char x[64]; } *p = __p; 242 243 asm volatile(ALTERNATIVE_2( 244 ".byte " __stringify(NOP_DS_PREFIX) "; clflush (%[pax])", 245 ".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */ 246 X86_FEATURE_CLFLUSHOPT, 247 ".byte 0x66, 0x0f, 0xae, 0x30", /* clwb (%%rax) */ 248 X86_FEATURE_CLWB) 249 : [p] "+m" (*p) 250 : [pax] "a" (p)); 251 } 252 253 #define nop() asm volatile ("nop") 254 255 256 #endif /* __KERNEL__ */ 257 258 #endif /* _ASM_X86_SPECIAL_INSNS_H */ 259