xref: /openbmc/linux/arch/x86/include/asm/nospec-branch.h (revision 05cf4fe738242183f1237f1b3a28b4479348c0a1)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 
3 #ifndef _ASM_X86_NOSPEC_BRANCH_H_
4 #define _ASM_X86_NOSPEC_BRANCH_H_
5 
6 #include <linux/static_key.h>
7 
8 #include <asm/alternative.h>
9 #include <asm/alternative-asm.h>
10 #include <asm/cpufeatures.h>
11 #include <asm/msr-index.h>
12 
13 /*
14  * Fill the CPU return stack buffer.
15  *
16  * Each entry in the RSB, if used for a speculative 'ret', contains an
17  * infinite 'pause; lfence; jmp' loop to capture speculative execution.
18  *
19  * This is required in various cases for retpoline and IBRS-based
20  * mitigations for the Spectre variant 2 vulnerability. Sometimes to
21  * eliminate potentially bogus entries from the RSB, and sometimes
22  * purely to ensure that it doesn't get empty, which on some CPUs would
23  * allow predictions from other (unwanted!) sources to be used.
24  *
25  * We define a CPP macro such that it can be used from both .S files and
26  * inline assembly. It's possible to do a .macro and then include that
27  * from C via asm(".include <asm/nospec-branch.h>") but let's not go there.
28  */
29 
30 #define RSB_CLEAR_LOOPS		32	/* To forcibly overwrite all entries */
31 #define RSB_FILL_LOOPS		16	/* To avoid underflow */
32 
33 /*
34  * Google experimented with loop-unrolling and this turned out to be
35  * the optimal version — two calls, each with their own speculation
36  * trap should their return address end up getting used, in a loop.
37  */
38 #define __FILL_RETURN_BUFFER(reg, nr, sp)	\
39 	mov	$(nr/2), reg;			\
40 771:						\
41 	call	772f;				\
42 773:	/* speculation trap */			\
43 	pause;					\
44 	lfence;					\
45 	jmp	773b;				\
46 772:						\
47 	call	774f;				\
48 775:	/* speculation trap */			\
49 	pause;					\
50 	lfence;					\
51 	jmp	775b;				\
52 774:						\
53 	dec	reg;				\
54 	jnz	771b;				\
55 	add	$(BITS_PER_LONG/8) * nr, sp;
56 
57 #ifdef __ASSEMBLY__
58 
59 /*
60  * This should be used immediately before a retpoline alternative.  It tells
61  * objtool where the retpolines are so that it can make sense of the control
62  * flow by just reading the original instruction(s) and ignoring the
63  * alternatives.
64  */
65 .macro ANNOTATE_NOSPEC_ALTERNATIVE
66 	.Lannotate_\@:
67 	.pushsection .discard.nospec
68 	.long .Lannotate_\@ - .
69 	.popsection
70 .endm
71 
72 /*
73  * This should be used immediately before an indirect jump/call. It tells
74  * objtool the subsequent indirect jump/call is vouched safe for retpoline
75  * builds.
76  */
77 .macro ANNOTATE_RETPOLINE_SAFE
78 	.Lannotate_\@:
79 	.pushsection .discard.retpoline_safe
80 	_ASM_PTR .Lannotate_\@
81 	.popsection
82 .endm
83 
84 /*
85  * These are the bare retpoline primitives for indirect jmp and call.
86  * Do not use these directly; they only exist to make the ALTERNATIVE
87  * invocation below less ugly.
88  */
89 .macro RETPOLINE_JMP reg:req
90 	call	.Ldo_rop_\@
91 .Lspec_trap_\@:
92 	pause
93 	lfence
94 	jmp	.Lspec_trap_\@
95 .Ldo_rop_\@:
96 	mov	\reg, (%_ASM_SP)
97 	ret
98 .endm
99 
100 /*
101  * This is a wrapper around RETPOLINE_JMP so the called function in reg
102  * returns to the instruction after the macro.
103  */
104 .macro RETPOLINE_CALL reg:req
105 	jmp	.Ldo_call_\@
106 .Ldo_retpoline_jmp_\@:
107 	RETPOLINE_JMP \reg
108 .Ldo_call_\@:
109 	call	.Ldo_retpoline_jmp_\@
110 .endm
111 
112 /*
113  * JMP_NOSPEC and CALL_NOSPEC macros can be used instead of a simple
114  * indirect jmp/call which may be susceptible to the Spectre variant 2
115  * attack.
116  */
117 .macro JMP_NOSPEC reg:req
118 #ifdef CONFIG_RETPOLINE
119 	ANNOTATE_NOSPEC_ALTERNATIVE
120 	ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg),	\
121 		__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE,	\
122 		__stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
123 #else
124 	jmp	*\reg
125 #endif
126 .endm
127 
128 .macro CALL_NOSPEC reg:req
129 #ifdef CONFIG_RETPOLINE
130 	ANNOTATE_NOSPEC_ALTERNATIVE
131 	ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg),	\
132 		__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
133 		__stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD
134 #else
135 	call	*\reg
136 #endif
137 .endm
138 
139  /*
140   * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP
141   * monstrosity above, manually.
142   */
143 .macro FILL_RETURN_BUFFER reg:req nr:req ftr:req
144 #ifdef CONFIG_RETPOLINE
145 	ANNOTATE_NOSPEC_ALTERNATIVE
146 	ALTERNATIVE "jmp .Lskip_rsb_\@",				\
147 		__stringify(__FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP))	\
148 		\ftr
149 .Lskip_rsb_\@:
150 #endif
151 .endm
152 
153 #else /* __ASSEMBLY__ */
154 
155 #define ANNOTATE_NOSPEC_ALTERNATIVE				\
156 	"999:\n\t"						\
157 	".pushsection .discard.nospec\n\t"			\
158 	".long 999b - .\n\t"					\
159 	".popsection\n\t"
160 
161 #define ANNOTATE_RETPOLINE_SAFE					\
162 	"999:\n\t"						\
163 	".pushsection .discard.retpoline_safe\n\t"		\
164 	_ASM_PTR " 999b\n\t"					\
165 	".popsection\n\t"
166 
167 #ifdef CONFIG_RETPOLINE
168 #ifdef CONFIG_X86_64
169 
170 /*
171  * Inline asm uses the %V modifier which is only in newer GCC
172  * which is ensured when CONFIG_RETPOLINE is defined.
173  */
174 # define CALL_NOSPEC						\
175 	ANNOTATE_NOSPEC_ALTERNATIVE				\
176 	ALTERNATIVE_2(						\
177 	ANNOTATE_RETPOLINE_SAFE					\
178 	"call *%[thunk_target]\n",				\
179 	"call __x86_indirect_thunk_%V[thunk_target]\n",		\
180 	X86_FEATURE_RETPOLINE,					\
181 	"lfence;\n"						\
182 	ANNOTATE_RETPOLINE_SAFE					\
183 	"call *%[thunk_target]\n",				\
184 	X86_FEATURE_RETPOLINE_AMD)
185 # define THUNK_TARGET(addr) [thunk_target] "r" (addr)
186 
187 #else /* CONFIG_X86_32 */
188 /*
189  * For i386 we use the original ret-equivalent retpoline, because
190  * otherwise we'll run out of registers. We don't care about CET
191  * here, anyway.
192  */
193 # define CALL_NOSPEC						\
194 	ANNOTATE_NOSPEC_ALTERNATIVE				\
195 	ALTERNATIVE_2(						\
196 	ANNOTATE_RETPOLINE_SAFE					\
197 	"call *%[thunk_target]\n",				\
198 	"       jmp    904f;\n"					\
199 	"       .align 16\n"					\
200 	"901:	call   903f;\n"					\
201 	"902:	pause;\n"					\
202 	"    	lfence;\n"					\
203 	"       jmp    902b;\n"					\
204 	"       .align 16\n"					\
205 	"903:	addl   $4, %%esp;\n"				\
206 	"       pushl  %[thunk_target];\n"			\
207 	"       ret;\n"						\
208 	"       .align 16\n"					\
209 	"904:	call   901b;\n",				\
210 	X86_FEATURE_RETPOLINE,					\
211 	"lfence;\n"						\
212 	ANNOTATE_RETPOLINE_SAFE					\
213 	"call *%[thunk_target]\n",				\
214 	X86_FEATURE_RETPOLINE_AMD)
215 
216 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
217 #endif
218 #else /* No retpoline for C / inline asm */
219 # define CALL_NOSPEC "call *%[thunk_target]\n"
220 # define THUNK_TARGET(addr) [thunk_target] "rm" (addr)
221 #endif
222 
223 /* The Spectre V2 mitigation variants */
224 enum spectre_v2_mitigation {
225 	SPECTRE_V2_NONE,
226 	SPECTRE_V2_RETPOLINE_GENERIC,
227 	SPECTRE_V2_RETPOLINE_AMD,
228 	SPECTRE_V2_IBRS_ENHANCED,
229 };
230 
231 /* The indirect branch speculation control variants */
232 enum spectre_v2_user_mitigation {
233 	SPECTRE_V2_USER_NONE,
234 	SPECTRE_V2_USER_STRICT,
235 	SPECTRE_V2_USER_PRCTL,
236 	SPECTRE_V2_USER_SECCOMP,
237 };
238 
239 /* The Speculative Store Bypass disable variants */
240 enum ssb_mitigation {
241 	SPEC_STORE_BYPASS_NONE,
242 	SPEC_STORE_BYPASS_DISABLE,
243 	SPEC_STORE_BYPASS_PRCTL,
244 	SPEC_STORE_BYPASS_SECCOMP,
245 };
246 
247 extern char __indirect_thunk_start[];
248 extern char __indirect_thunk_end[];
249 
250 /*
251  * On VMEXIT we must ensure that no RSB predictions learned in the guest
252  * can be followed in the host, by overwriting the RSB completely. Both
253  * retpoline and IBRS mitigations for Spectre v2 need this; only on future
254  * CPUs with IBRS_ALL *might* it be avoided.
255  */
256 static inline void vmexit_fill_RSB(void)
257 {
258 #ifdef CONFIG_RETPOLINE
259 	unsigned long loops;
260 
261 	asm volatile (ANNOTATE_NOSPEC_ALTERNATIVE
262 		      ALTERNATIVE("jmp 910f",
263 				  __stringify(__FILL_RETURN_BUFFER(%0, RSB_CLEAR_LOOPS, %1)),
264 				  X86_FEATURE_RETPOLINE)
265 		      "910:"
266 		      : "=r" (loops), ASM_CALL_CONSTRAINT
267 		      : : "memory" );
268 #endif
269 }
270 
271 static __always_inline
272 void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
273 {
274 	asm volatile(ALTERNATIVE("", "wrmsr", %c[feature])
275 		: : "c" (msr),
276 		    "a" ((u32)val),
277 		    "d" ((u32)(val >> 32)),
278 		    [feature] "i" (feature)
279 		: "memory");
280 }
281 
282 static inline void indirect_branch_prediction_barrier(void)
283 {
284 	u64 val = PRED_CMD_IBPB;
285 
286 	alternative_msr_write(MSR_IA32_PRED_CMD, val, X86_FEATURE_USE_IBPB);
287 }
288 
289 /* The Intel SPEC CTRL MSR base value cache */
290 extern u64 x86_spec_ctrl_base;
291 
292 /*
293  * With retpoline, we must use IBRS to restrict branch prediction
294  * before calling into firmware.
295  *
296  * (Implemented as CPP macros due to header hell.)
297  */
298 #define firmware_restrict_branch_speculation_start()			\
299 do {									\
300 	u64 val = x86_spec_ctrl_base | SPEC_CTRL_IBRS;			\
301 									\
302 	preempt_disable();						\
303 	alternative_msr_write(MSR_IA32_SPEC_CTRL, val,			\
304 			      X86_FEATURE_USE_IBRS_FW);			\
305 } while (0)
306 
307 #define firmware_restrict_branch_speculation_end()			\
308 do {									\
309 	u64 val = x86_spec_ctrl_base;					\
310 									\
311 	alternative_msr_write(MSR_IA32_SPEC_CTRL, val,			\
312 			      X86_FEATURE_USE_IBRS_FW);			\
313 	preempt_enable();						\
314 } while (0)
315 
316 DECLARE_STATIC_KEY_FALSE(switch_to_cond_stibp);
317 DECLARE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
318 DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
319 
320 #endif /* __ASSEMBLY__ */
321 
322 /*
323  * Below is used in the eBPF JIT compiler and emits the byte sequence
324  * for the following assembly:
325  *
326  * With retpolines configured:
327  *
328  *    callq do_rop
329  *  spec_trap:
330  *    pause
331  *    lfence
332  *    jmp spec_trap
333  *  do_rop:
334  *    mov %rax,(%rsp) for x86_64
335  *    mov %edx,(%esp) for x86_32
336  *    retq
337  *
338  * Without retpolines configured:
339  *
340  *    jmp *%rax for x86_64
341  *    jmp *%edx for x86_32
342  */
343 #ifdef CONFIG_RETPOLINE
344 # ifdef CONFIG_X86_64
345 #  define RETPOLINE_RAX_BPF_JIT_SIZE	17
346 #  define RETPOLINE_RAX_BPF_JIT()				\
347 do {								\
348 	EMIT1_off32(0xE8, 7);	 /* callq do_rop */		\
349 	/* spec_trap: */					\
350 	EMIT2(0xF3, 0x90);       /* pause */			\
351 	EMIT3(0x0F, 0xAE, 0xE8); /* lfence */			\
352 	EMIT2(0xEB, 0xF9);       /* jmp spec_trap */		\
353 	/* do_rop: */						\
354 	EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */	\
355 	EMIT1(0xC3);             /* retq */			\
356 } while (0)
357 # else /* !CONFIG_X86_64 */
358 #  define RETPOLINE_EDX_BPF_JIT()				\
359 do {								\
360 	EMIT1_off32(0xE8, 7);	 /* call do_rop */		\
361 	/* spec_trap: */					\
362 	EMIT2(0xF3, 0x90);       /* pause */			\
363 	EMIT3(0x0F, 0xAE, 0xE8); /* lfence */			\
364 	EMIT2(0xEB, 0xF9);       /* jmp spec_trap */		\
365 	/* do_rop: */						\
366 	EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */		\
367 	EMIT1(0xC3);             /* ret */			\
368 } while (0)
369 # endif
370 #else /* !CONFIG_RETPOLINE */
371 # ifdef CONFIG_X86_64
372 #  define RETPOLINE_RAX_BPF_JIT_SIZE	2
373 #  define RETPOLINE_RAX_BPF_JIT()				\
374 	EMIT2(0xFF, 0xE0);       /* jmp *%rax */
375 # else /* !CONFIG_X86_64 */
376 #  define RETPOLINE_EDX_BPF_JIT()				\
377 	EMIT2(0xFF, 0xE2)        /* jmp *%edx */
378 # endif
379 #endif
380 
381 #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
382