1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_ALTERNATIVE_H
3 #define _ASM_X86_ALTERNATIVE_H
4
5 #include <linux/types.h>
6 #include <linux/stringify.h>
7 #include <asm/asm.h>
8
9 #define ALT_FLAGS_SHIFT 16
10
11 #define ALT_FLAG_NOT (1 << 0)
12 #define ALT_NOT(feature) ((ALT_FLAG_NOT << ALT_FLAGS_SHIFT) | (feature))
13
14 #ifndef __ASSEMBLY__
15
16 #include <linux/stddef.h>
17
18 /*
19 * Alternative inline assembly for SMP.
20 *
21 * The LOCK_PREFIX macro defined here replaces the LOCK and
22 * LOCK_PREFIX macros used everywhere in the source tree.
23 *
24 * SMP alternatives use the same data structures as the other
25 * alternatives and the X86_FEATURE_UP flag to indicate the case of a
26 * UP system running a SMP kernel. The existing apply_alternatives()
27 * works fine for patching a SMP kernel for UP.
28 *
29 * The SMP alternative tables can be kept after boot and contain both
30 * UP and SMP versions of the instructions to allow switching back to
31 * SMP at runtime, when hotplugging in a new CPU, which is especially
32 * useful in virtualized environments.
33 *
34 * The very common lock prefix is handled as special case in a
35 * separate table which is a pure address list without replacement ptr
36 * and size information. That keeps the table sizes small.
37 */
38
39 #ifdef CONFIG_SMP
40 #define LOCK_PREFIX_HERE \
41 ".pushsection .smp_locks,\"a\"\n" \
42 ".balign 4\n" \
43 ".long 671f - .\n" /* offset */ \
44 ".popsection\n" \
45 "671:"
46
47 #define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
48
49 #else /* ! CONFIG_SMP */
50 #define LOCK_PREFIX_HERE ""
51 #define LOCK_PREFIX ""
52 #endif
53
54 /*
55 * objtool annotation to ignore the alternatives and only consider the original
56 * instruction(s).
57 */
58 #define ANNOTATE_IGNORE_ALTERNATIVE \
59 "999:\n\t" \
60 ".pushsection .discard.ignore_alts\n\t" \
61 ".long 999b\n\t" \
62 ".popsection\n\t"
63
64 /*
65 * The patching flags are part of the upper bits of the @ft_flags parameter when
66 * specifying them. The split is currently like this:
67 *
68 * [31... flags ...16][15... CPUID feature bit ...0]
69 *
70 * but since this is all hidden in the macros argument being split, those fields can be
71 * extended in the future to fit in a u64 or however the need arises.
72 */
73 struct alt_instr {
74 s32 instr_offset; /* original instruction */
75 s32 repl_offset; /* offset to replacement instruction */
76
77 union {
78 struct {
79 u32 cpuid: 16; /* CPUID bit set for replacement */
80 u32 flags: 16; /* patching control flags */
81 };
82 u32 ft_flags;
83 };
84
85 u8 instrlen; /* length of original instruction */
86 u8 replacementlen; /* length of new instruction */
87 } __packed;
88
89 /*
90 * Debug flag that can be tested to see whether alternative
91 * instructions were patched in already:
92 */
93 extern int alternatives_patched;
94
95 extern void alternative_instructions(void);
96 extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
97 extern void apply_retpolines(s32 *start, s32 *end);
98 extern void apply_returns(s32 *start, s32 *end);
99 extern void apply_seal_endbr(s32 *start, s32 *end);
100 extern void apply_fineibt(s32 *start_retpoline, s32 *end_retpoine,
101 s32 *start_cfi, s32 *end_cfi);
102
103 struct module;
104 struct paravirt_patch_site;
105
106 struct callthunk_sites {
107 s32 *call_start, *call_end;
108 struct paravirt_patch_site *pv_start, *pv_end;
109 };
110
111 #ifdef CONFIG_CALL_THUNKS
112 extern void callthunks_patch_builtin_calls(void);
113 extern void callthunks_patch_module_calls(struct callthunk_sites *sites,
114 struct module *mod);
115 extern void *callthunks_translate_call_dest(void *dest);
116 extern int x86_call_depth_emit_accounting(u8 **pprog, void *func);
117 #else
callthunks_patch_builtin_calls(void)118 static __always_inline void callthunks_patch_builtin_calls(void) {}
119 static __always_inline void
callthunks_patch_module_calls(struct callthunk_sites * sites,struct module * mod)120 callthunks_patch_module_calls(struct callthunk_sites *sites,
121 struct module *mod) {}
callthunks_translate_call_dest(void * dest)122 static __always_inline void *callthunks_translate_call_dest(void *dest)
123 {
124 return dest;
125 }
x86_call_depth_emit_accounting(u8 ** pprog,void * func)126 static __always_inline int x86_call_depth_emit_accounting(u8 **pprog,
127 void *func)
128 {
129 return 0;
130 }
131 #endif
132
133 #ifdef CONFIG_SMP
134 extern void alternatives_smp_module_add(struct module *mod, char *name,
135 void *locks, void *locks_end,
136 void *text, void *text_end);
137 extern void alternatives_smp_module_del(struct module *mod);
138 extern void alternatives_enable_smp(void);
139 extern int alternatives_text_reserved(void *start, void *end);
140 extern bool skip_smp_alternatives;
141 #else
alternatives_smp_module_add(struct module * mod,char * name,void * locks,void * locks_end,void * text,void * text_end)142 static inline void alternatives_smp_module_add(struct module *mod, char *name,
143 void *locks, void *locks_end,
144 void *text, void *text_end) {}
alternatives_smp_module_del(struct module * mod)145 static inline void alternatives_smp_module_del(struct module *mod) {}
alternatives_enable_smp(void)146 static inline void alternatives_enable_smp(void) {}
alternatives_text_reserved(void * start,void * end)147 static inline int alternatives_text_reserved(void *start, void *end)
148 {
149 return 0;
150 }
151 #endif /* CONFIG_SMP */
152
153 #define b_replacement(num) "664"#num
154 #define e_replacement(num) "665"#num
155
156 #define alt_end_marker "663"
157 #define alt_slen "662b-661b"
158 #define alt_total_slen alt_end_marker"b-661b"
159 #define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f"
160
161 #define OLDINSTR(oldinstr, num) \
162 "# ALT: oldnstr\n" \
163 "661:\n\t" oldinstr "\n662:\n" \
164 "# ALT: padding\n" \
165 ".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \
166 "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n" \
167 alt_end_marker ":\n"
168
169 /*
170 * gas compatible max based on the idea from:
171 * http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
172 *
173 * The additional "-" is needed because gas uses a "true" value of -1.
174 */
175 #define alt_max_short(a, b) "((" a ") ^ (((" a ") ^ (" b ")) & -(-((" a ") < (" b ")))))"
176
177 /*
178 * Pad the second replacement alternative with additional NOPs if it is
179 * additionally longer than the first replacement alternative.
180 */
181 #define OLDINSTR_2(oldinstr, num1, num2) \
182 "# ALT: oldinstr2\n" \
183 "661:\n\t" oldinstr "\n662:\n" \
184 "# ALT: padding2\n" \
185 ".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * " \
186 "(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")), 0x90\n" \
187 alt_end_marker ":\n"
188
189 #define OLDINSTR_3(oldinsn, n1, n2, n3) \
190 "# ALT: oldinstr3\n" \
191 "661:\n\t" oldinsn "\n662:\n" \
192 "# ALT: padding3\n" \
193 ".skip -((" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3)) \
194 " - (" alt_slen ")) > 0) * " \
195 "(" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3)) \
196 " - (" alt_slen ")), 0x90\n" \
197 alt_end_marker ":\n"
198
199 #define ALTINSTR_ENTRY(ft_flags, num) \
200 " .long 661b - .\n" /* label */ \
201 " .long " b_replacement(num)"f - .\n" /* new instruction */ \
202 " .4byte " __stringify(ft_flags) "\n" /* feature + flags */ \
203 " .byte " alt_total_slen "\n" /* source len */ \
204 " .byte " alt_rlen(num) "\n" /* replacement len */
205
206 #define ALTINSTR_REPLACEMENT(newinstr, num) /* replacement */ \
207 "# ALT: replacement " #num "\n" \
208 b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n"
209
210 /* alternative assembly primitive: */
211 #define ALTERNATIVE(oldinstr, newinstr, ft_flags) \
212 OLDINSTR(oldinstr, 1) \
213 ".pushsection .altinstructions,\"a\"\n" \
214 ALTINSTR_ENTRY(ft_flags, 1) \
215 ".popsection\n" \
216 ".pushsection .altinstr_replacement, \"ax\"\n" \
217 ALTINSTR_REPLACEMENT(newinstr, 1) \
218 ".popsection\n"
219
220 #define ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \
221 OLDINSTR_2(oldinstr, 1, 2) \
222 ".pushsection .altinstructions,\"a\"\n" \
223 ALTINSTR_ENTRY(ft_flags1, 1) \
224 ALTINSTR_ENTRY(ft_flags2, 2) \
225 ".popsection\n" \
226 ".pushsection .altinstr_replacement, \"ax\"\n" \
227 ALTINSTR_REPLACEMENT(newinstr1, 1) \
228 ALTINSTR_REPLACEMENT(newinstr2, 2) \
229 ".popsection\n"
230
231 /* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */
232 #define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \
233 ALTERNATIVE_2(oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \
234 newinstr_yes, ft_flags)
235
236 #define ALTERNATIVE_3(oldinsn, newinsn1, ft_flags1, newinsn2, ft_flags2, \
237 newinsn3, ft_flags3) \
238 OLDINSTR_3(oldinsn, 1, 2, 3) \
239 ".pushsection .altinstructions,\"a\"\n" \
240 ALTINSTR_ENTRY(ft_flags1, 1) \
241 ALTINSTR_ENTRY(ft_flags2, 2) \
242 ALTINSTR_ENTRY(ft_flags3, 3) \
243 ".popsection\n" \
244 ".pushsection .altinstr_replacement, \"ax\"\n" \
245 ALTINSTR_REPLACEMENT(newinsn1, 1) \
246 ALTINSTR_REPLACEMENT(newinsn2, 2) \
247 ALTINSTR_REPLACEMENT(newinsn3, 3) \
248 ".popsection\n"
249
250 /*
251 * Alternative instructions for different CPU types or capabilities.
252 *
253 * This allows to use optimized instructions even on generic binary
254 * kernels.
255 *
256 * length of oldinstr must be longer or equal the length of newinstr
257 * It can be padded with nops as needed.
258 *
259 * For non barrier like inlines please define new variants
260 * without volatile and memory clobber.
261 */
262 #define alternative(oldinstr, newinstr, ft_flags) \
263 asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) : : : "memory")
264
265 #define alternative_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \
266 asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) ::: "memory")
267
268 #define alternative_ternary(oldinstr, ft_flags, newinstr_yes, newinstr_no) \
269 asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) ::: "memory")
270
271 /*
272 * Alternative inline assembly with input.
273 *
274 * Peculiarities:
275 * No memory clobber here.
276 * Argument numbers start with 1.
277 * Leaving an unused argument 0 to keep API compatibility.
278 */
279 #define alternative_input(oldinstr, newinstr, ft_flags, input...) \
280 asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \
281 : : "i" (0), ## input)
282
283 /*
284 * This is similar to alternative_input. But it has two features and
285 * respective instructions.
286 *
287 * If CPU has feature2, newinstr2 is used.
288 * Otherwise, if CPU has feature1, newinstr1 is used.
289 * Otherwise, oldinstr is used.
290 */
291 #define alternative_input_2(oldinstr, newinstr1, ft_flags1, newinstr2, \
292 ft_flags2, input...) \
293 asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, \
294 newinstr2, ft_flags2) \
295 : : "i" (0), ## input)
296
297 /* Like alternative_input, but with a single output argument */
298 #define alternative_io(oldinstr, newinstr, ft_flags, output, input...) \
299 asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \
300 : output : "i" (0), ## input)
301
302 /* Like alternative_io, but for replacing a direct call with another one. */
303 #define alternative_call(oldfunc, newfunc, ft_flags, output, input...) \
304 asm_inline volatile (ALTERNATIVE("call %c[old]", "call %c[new]", ft_flags) \
305 : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
306
307 /*
308 * Like alternative_call, but there are two features and respective functions.
309 * If CPU has feature2, function2 is used.
310 * Otherwise, if CPU has feature1, function1 is used.
311 * Otherwise, old function is used.
312 */
313 #define alternative_call_2(oldfunc, newfunc1, ft_flags1, newfunc2, ft_flags2, \
314 output, input...) \
315 asm_inline volatile (ALTERNATIVE_2("call %c[old]", "call %c[new1]", ft_flags1, \
316 "call %c[new2]", ft_flags2) \
317 : output, ASM_CALL_CONSTRAINT \
318 : [old] "i" (oldfunc), [new1] "i" (newfunc1), \
319 [new2] "i" (newfunc2), ## input)
320
321 /*
322 * use this macro(s) if you need more than one output parameter
323 * in alternative_io
324 */
325 #define ASM_OUTPUT2(a...) a
326
327 /*
328 * use this macro if you need clobbers but no inputs in
329 * alternative_{input,io,call}()
330 */
331 #define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
332
333 #else /* __ASSEMBLY__ */
334
335 #ifdef CONFIG_SMP
336 .macro LOCK_PREFIX
337 672: lock
338 .pushsection .smp_locks,"a"
339 .balign 4
340 .long 672b - .
341 .popsection
342 .endm
343 #else
344 .macro LOCK_PREFIX
345 .endm
346 #endif
347
348 /*
349 * objtool annotation to ignore the alternatives and only consider the original
350 * instruction(s).
351 */
352 .macro ANNOTATE_IGNORE_ALTERNATIVE
353 .Lannotate_\@:
354 .pushsection .discard.ignore_alts
355 .long .Lannotate_\@
356 .popsection
357 .endm
358
359 /*
360 * Issue one struct alt_instr descriptor entry (need to put it into
361 * the section .altinstructions, see below). This entry contains
362 * enough information for the alternatives patching code to patch an
363 * instruction. See apply_alternatives().
364 */
365 .macro altinstr_entry orig alt ft_flags orig_len alt_len
366 .long \orig - .
367 .long \alt - .
368 .4byte \ft_flags
369 .byte \orig_len
370 .byte \alt_len
371 .endm
372
373 /*
374 * Define an alternative between two instructions. If @feature is
375 * present, early code in apply_alternatives() replaces @oldinstr with
376 * @newinstr. ".skip" directive takes care of proper instruction padding
377 * in case @newinstr is longer than @oldinstr.
378 */
379 .macro ALTERNATIVE oldinstr, newinstr, ft_flags
380 140:
381 \oldinstr
382 141:
383 .skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90
384 142:
385
386 .pushsection .altinstructions,"a"
387 altinstr_entry 140b,143f,\ft_flags,142b-140b,144f-143f
388 .popsection
389
390 .pushsection .altinstr_replacement,"ax"
391 143:
392 \newinstr
393 144:
394 .popsection
395 .endm
396
397 #define old_len 141b-140b
398 #define new_len1 144f-143f
399 #define new_len2 145f-144f
400 #define new_len3 146f-145f
401
402 /*
403 * gas compatible max based on the idea from:
404 * http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
405 *
406 * The additional "-" is needed because gas uses a "true" value of -1.
407 */
408 #define alt_max_2(a, b) ((a) ^ (((a) ^ (b)) & -(-((a) < (b)))))
409 #define alt_max_3(a, b, c) (alt_max_2(alt_max_2(a, b), c))
410
411
412 /*
413 * Same as ALTERNATIVE macro above but for two alternatives. If CPU
414 * has @feature1, it replaces @oldinstr with @newinstr1. If CPU has
415 * @feature2, it replaces @oldinstr with @feature2.
416 */
417 .macro ALTERNATIVE_2 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2
418 140:
419 \oldinstr
420 141:
421 .skip -((alt_max_2(new_len1, new_len2) - (old_len)) > 0) * \
422 (alt_max_2(new_len1, new_len2) - (old_len)),0x90
423 142:
424
425 .pushsection .altinstructions,"a"
426 altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f
427 altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f
428 .popsection
429
430 .pushsection .altinstr_replacement,"ax"
431 143:
432 \newinstr1
433 144:
434 \newinstr2
435 145:
436 .popsection
437 .endm
438
439 .macro ALTERNATIVE_3 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2, newinstr3, ft_flags3
440 140:
441 \oldinstr
442 141:
443 .skip -((alt_max_3(new_len1, new_len2, new_len3) - (old_len)) > 0) * \
444 (alt_max_3(new_len1, new_len2, new_len3) - (old_len)),0x90
445 142:
446
447 .pushsection .altinstructions,"a"
448 altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f
449 altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f
450 altinstr_entry 140b,145f,\ft_flags3,142b-140b,146f-145f
451 .popsection
452
453 .pushsection .altinstr_replacement,"ax"
454 143:
455 \newinstr1
456 144:
457 \newinstr2
458 145:
459 \newinstr3
460 146:
461 .popsection
462 .endm
463
464 /* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */
465 #define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \
466 ALTERNATIVE_2 oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \
467 newinstr_yes, ft_flags
468
469 #endif /* __ASSEMBLY__ */
470
471 #endif /* _ASM_X86_ALTERNATIVE_H */
472