1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_SPECIAL_INSNS_H 3 #define _ASM_X86_SPECIAL_INSNS_H 4 5 6 #ifdef __KERNEL__ 7 8 #include <asm/nops.h> 9 #include <asm/processor-flags.h> 10 #include <linux/jump_label.h> 11 12 /* 13 * Volatile isn't enough to prevent the compiler from reordering the 14 * read/write functions for the control registers and messing everything up. 15 * A memory clobber would solve the problem, but would prevent reordering of 16 * all loads stores around it, which can hurt performance. Solution is to 17 * use a variable and mimic reads and writes to it to enforce serialization 18 */ 19 extern unsigned long __force_order; 20 21 void native_write_cr0(unsigned long val); 22 23 static inline unsigned long native_read_cr0(void) 24 { 25 unsigned long val; 26 asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order)); 27 return val; 28 } 29 30 static inline unsigned long native_read_cr2(void) 31 { 32 unsigned long val; 33 asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order)); 34 return val; 35 } 36 37 static inline void native_write_cr2(unsigned long val) 38 { 39 asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order)); 40 } 41 42 static inline unsigned long __native_read_cr3(void) 43 { 44 unsigned long val; 45 asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order)); 46 return val; 47 } 48 49 static inline void native_write_cr3(unsigned long val) 50 { 51 asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order)); 52 } 53 54 static inline unsigned long native_read_cr4(void) 55 { 56 unsigned long val; 57 #ifdef CONFIG_X86_32 58 /* 59 * This could fault if CR4 does not exist. Non-existent CR4 60 * is functionally equivalent to CR4 == 0. Keep it simple and pretend 61 * that CR4 == 0 on CPUs that don't have CR4. 62 */ 63 asm volatile("1: mov %%cr4, %0\n" 64 "2:\n" 65 _ASM_EXTABLE(1b, 2b) 66 : "=r" (val), "=m" (__force_order) : "0" (0)); 67 #else 68 /* CR4 always exists on x86_64. */ 69 asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order)); 70 #endif 71 return val; 72 } 73 74 void native_write_cr4(unsigned long val); 75 76 #ifdef CONFIG_X86_64 77 static inline unsigned long native_read_cr8(void) 78 { 79 unsigned long cr8; 80 asm volatile("movq %%cr8,%0" : "=r" (cr8)); 81 return cr8; 82 } 83 84 static inline void native_write_cr8(unsigned long val) 85 { 86 asm volatile("movq %0,%%cr8" :: "r" (val) : "memory"); 87 } 88 #endif 89 90 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS 91 static inline u32 rdpkru(void) 92 { 93 u32 ecx = 0; 94 u32 edx, pkru; 95 96 /* 97 * "rdpkru" instruction. Places PKRU contents in to EAX, 98 * clears EDX and requires that ecx=0. 99 */ 100 asm volatile(".byte 0x0f,0x01,0xee\n\t" 101 : "=a" (pkru), "=d" (edx) 102 : "c" (ecx)); 103 return pkru; 104 } 105 106 static inline void wrpkru(u32 pkru) 107 { 108 u32 ecx = 0, edx = 0; 109 110 /* 111 * "wrpkru" instruction. Loads contents in EAX to PKRU, 112 * requires that ecx = edx = 0. 113 */ 114 asm volatile(".byte 0x0f,0x01,0xef\n\t" 115 : : "a" (pkru), "c"(ecx), "d"(edx)); 116 } 117 118 static inline void __write_pkru(u32 pkru) 119 { 120 /* 121 * WRPKRU is relatively expensive compared to RDPKRU. 122 * Avoid WRPKRU when it would not change the value. 123 */ 124 if (pkru == rdpkru()) 125 return; 126 127 wrpkru(pkru); 128 } 129 130 #else 131 static inline u32 rdpkru(void) 132 { 133 return 0; 134 } 135 136 static inline void __write_pkru(u32 pkru) 137 { 138 } 139 #endif 140 141 static inline void native_wbinvd(void) 142 { 143 asm volatile("wbinvd": : :"memory"); 144 } 145 146 extern asmlinkage void native_load_gs_index(unsigned); 147 148 static inline unsigned long __read_cr4(void) 149 { 150 return native_read_cr4(); 151 } 152 153 #ifdef CONFIG_PARAVIRT_XXL 154 #include <asm/paravirt.h> 155 #else 156 157 static inline unsigned long read_cr0(void) 158 { 159 return native_read_cr0(); 160 } 161 162 static inline void write_cr0(unsigned long x) 163 { 164 native_write_cr0(x); 165 } 166 167 static inline unsigned long read_cr2(void) 168 { 169 return native_read_cr2(); 170 } 171 172 static inline void write_cr2(unsigned long x) 173 { 174 native_write_cr2(x); 175 } 176 177 /* 178 * Careful! CR3 contains more than just an address. You probably want 179 * read_cr3_pa() instead. 180 */ 181 static inline unsigned long __read_cr3(void) 182 { 183 return __native_read_cr3(); 184 } 185 186 static inline void write_cr3(unsigned long x) 187 { 188 native_write_cr3(x); 189 } 190 191 static inline void __write_cr4(unsigned long x) 192 { 193 native_write_cr4(x); 194 } 195 196 static inline void wbinvd(void) 197 { 198 native_wbinvd(); 199 } 200 201 #ifdef CONFIG_X86_64 202 203 static inline unsigned long read_cr8(void) 204 { 205 return native_read_cr8(); 206 } 207 208 static inline void write_cr8(unsigned long x) 209 { 210 native_write_cr8(x); 211 } 212 213 static inline void load_gs_index(unsigned selector) 214 { 215 native_load_gs_index(selector); 216 } 217 218 #endif 219 220 #endif /* CONFIG_PARAVIRT_XXL */ 221 222 static inline void clflush(volatile void *__p) 223 { 224 asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p)); 225 } 226 227 static inline void clflushopt(volatile void *__p) 228 { 229 alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0", 230 ".byte 0x66; clflush %P0", 231 X86_FEATURE_CLFLUSHOPT, 232 "+m" (*(volatile char __force *)__p)); 233 } 234 235 static inline void clwb(volatile void *__p) 236 { 237 volatile struct { char x[64]; } *p = __p; 238 239 asm volatile(ALTERNATIVE_2( 240 ".byte " __stringify(NOP_DS_PREFIX) "; clflush (%[pax])", 241 ".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */ 242 X86_FEATURE_CLFLUSHOPT, 243 ".byte 0x66, 0x0f, 0xae, 0x30", /* clwb (%%rax) */ 244 X86_FEATURE_CLWB) 245 : [p] "+m" (*p) 246 : [pax] "a" (p)); 247 } 248 249 #define nop() asm volatile ("nop") 250 251 252 #endif /* __KERNEL__ */ 253 254 #endif /* _ASM_X86_SPECIAL_INSNS_H */ 255