1 #ifndef _ASM_X86_SPECIAL_INSNS_H
2 #define _ASM_X86_SPECIAL_INSNS_H
3 
4 
5 #ifdef __KERNEL__
6 
7 #include <asm/nops.h>
8 
9 static inline void native_clts(void)
10 {
11 	asm volatile("clts");
12 }
13 
14 /*
15  * Volatile isn't enough to prevent the compiler from reordering the
16  * read/write functions for the control registers and messing everything up.
17  * A memory clobber would solve the problem, but would prevent reordering of
18  * all loads stores around it, which can hurt performance. Solution is to
19  * use a variable and mimic reads and writes to it to enforce serialization
20  */
21 extern unsigned long __force_order;
22 
23 static inline unsigned long native_read_cr0(void)
24 {
25 	unsigned long val;
26 	asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order));
27 	return val;
28 }
29 
30 static inline void native_write_cr0(unsigned long val)
31 {
32 	asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order));
33 }
34 
35 static inline unsigned long native_read_cr2(void)
36 {
37 	unsigned long val;
38 	asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order));
39 	return val;
40 }
41 
42 static inline void native_write_cr2(unsigned long val)
43 {
44 	asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order));
45 }
46 
47 static inline unsigned long native_read_cr3(void)
48 {
49 	unsigned long val;
50 	asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order));
51 	return val;
52 }
53 
54 static inline void native_write_cr3(unsigned long val)
55 {
56 	asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order));
57 }
58 
59 static inline unsigned long native_read_cr4(void)
60 {
61 	unsigned long val;
62 	asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order));
63 	return val;
64 }
65 
66 static inline unsigned long native_read_cr4_safe(void)
67 {
68 	unsigned long val;
69 	/* This could fault if %cr4 does not exist. In x86_64, a cr4 always
70 	 * exists, so it will never fail. */
71 #ifdef CONFIG_X86_32
72 	asm volatile("1: mov %%cr4, %0\n"
73 		     "2:\n"
74 		     _ASM_EXTABLE(1b, 2b)
75 		     : "=r" (val), "=m" (__force_order) : "0" (0));
76 #else
77 	val = native_read_cr4();
78 #endif
79 	return val;
80 }
81 
82 static inline void native_write_cr4(unsigned long val)
83 {
84 	asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order));
85 }
86 
87 #ifdef CONFIG_X86_64
88 static inline unsigned long native_read_cr8(void)
89 {
90 	unsigned long cr8;
91 	asm volatile("movq %%cr8,%0" : "=r" (cr8));
92 	return cr8;
93 }
94 
95 static inline void native_write_cr8(unsigned long val)
96 {
97 	asm volatile("movq %0,%%cr8" :: "r" (val) : "memory");
98 }
99 #endif
100 
101 static inline void native_wbinvd(void)
102 {
103 	asm volatile("wbinvd": : :"memory");
104 }
105 
106 extern asmlinkage void native_load_gs_index(unsigned);
107 
108 #ifdef CONFIG_PARAVIRT
109 #include <asm/paravirt.h>
110 #else
111 
112 static inline unsigned long read_cr0(void)
113 {
114 	return native_read_cr0();
115 }
116 
117 static inline void write_cr0(unsigned long x)
118 {
119 	native_write_cr0(x);
120 }
121 
122 static inline unsigned long read_cr2(void)
123 {
124 	return native_read_cr2();
125 }
126 
127 static inline void write_cr2(unsigned long x)
128 {
129 	native_write_cr2(x);
130 }
131 
132 static inline unsigned long read_cr3(void)
133 {
134 	return native_read_cr3();
135 }
136 
137 static inline void write_cr3(unsigned long x)
138 {
139 	native_write_cr3(x);
140 }
141 
142 static inline unsigned long __read_cr4(void)
143 {
144 	return native_read_cr4();
145 }
146 
147 static inline unsigned long __read_cr4_safe(void)
148 {
149 	return native_read_cr4_safe();
150 }
151 
152 static inline void __write_cr4(unsigned long x)
153 {
154 	native_write_cr4(x);
155 }
156 
157 static inline void wbinvd(void)
158 {
159 	native_wbinvd();
160 }
161 
162 #ifdef CONFIG_X86_64
163 
164 static inline unsigned long read_cr8(void)
165 {
166 	return native_read_cr8();
167 }
168 
169 static inline void write_cr8(unsigned long x)
170 {
171 	native_write_cr8(x);
172 }
173 
174 static inline void load_gs_index(unsigned selector)
175 {
176 	native_load_gs_index(selector);
177 }
178 
179 #endif
180 
181 /* Clear the 'TS' bit */
182 static inline void clts(void)
183 {
184 	native_clts();
185 }
186 
187 #endif/* CONFIG_PARAVIRT */
188 
189 #define stts() write_cr0(read_cr0() | X86_CR0_TS)
190 
191 static inline void clflush(volatile void *__p)
192 {
193 	asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p));
194 }
195 
196 static inline void clflushopt(volatile void *__p)
197 {
198 	alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0",
199 		       ".byte 0x66; clflush %P0",
200 		       X86_FEATURE_CLFLUSHOPT,
201 		       "+m" (*(volatile char __force *)__p));
202 }
203 
204 static inline void clwb(volatile void *__p)
205 {
206 	volatile struct { char x[64]; } *p = __p;
207 
208 	asm volatile(ALTERNATIVE_2(
209 		".byte " __stringify(NOP_DS_PREFIX) "; clflush (%[pax])",
210 		".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */
211 		X86_FEATURE_CLFLUSHOPT,
212 		".byte 0x66, 0x0f, 0xae, 0x30",  /* clwb (%%rax) */
213 		X86_FEATURE_CLWB)
214 		: [p] "+m" (*p)
215 		: [pax] "a" (p));
216 }
217 
218 /**
219  * pcommit_sfence() - persistent commit and fence
220  *
221  * The PCOMMIT instruction ensures that data that has been flushed from the
222  * processor's cache hierarchy with CLWB, CLFLUSHOPT or CLFLUSH is accepted to
223  * memory and is durable on the DIMM.  The primary use case for this is
224  * persistent memory.
225  *
226  * This function shows how to properly use CLWB/CLFLUSHOPT/CLFLUSH and PCOMMIT
227  * with appropriate fencing.
228  *
229  * Example:
230  * void flush_and_commit_buffer(void *vaddr, unsigned int size)
231  * {
232  *         unsigned long clflush_mask = boot_cpu_data.x86_clflush_size - 1;
233  *         void *vend = vaddr + size;
234  *         void *p;
235  *
236  *         for (p = (void *)((unsigned long)vaddr & ~clflush_mask);
237  *              p < vend; p += boot_cpu_data.x86_clflush_size)
238  *                 clwb(p);
239  *
240  *         // SFENCE to order CLWB/CLFLUSHOPT/CLFLUSH cache flushes
241  *         // MFENCE via mb() also works
242  *         wmb();
243  *
244  *         // PCOMMIT and the required SFENCE for ordering
245  *         pcommit_sfence();
246  * }
247  *
248  * After this function completes the data pointed to by 'vaddr' has been
249  * accepted to memory and will be durable if the 'vaddr' points to persistent
250  * memory.
251  *
252  * PCOMMIT must always be ordered by an MFENCE or SFENCE, so to help simplify
253  * things we include both the PCOMMIT and the required SFENCE in the
254  * alternatives generated by pcommit_sfence().
255  */
256 static inline void pcommit_sfence(void)
257 {
258 	alternative(ASM_NOP7,
259 		    ".byte 0x66, 0x0f, 0xae, 0xf8\n\t" /* pcommit */
260 		    "sfence",
261 		    X86_FEATURE_PCOMMIT);
262 }
263 
264 #define nop() asm volatile ("nop")
265 
266 
267 #endif /* __KERNEL__ */
268 
269 #endif /* _ASM_X86_SPECIAL_INSNS_H */
270