1 #ifndef _ASM_X86_ALTERNATIVE_H 2 #define _ASM_X86_ALTERNATIVE_H 3 4 #include <linux/types.h> 5 #include <linux/stddef.h> 6 #include <linux/stringify.h> 7 #include <asm/asm.h> 8 #include <asm/ptrace.h> 9 10 /* 11 * Alternative inline assembly for SMP. 12 * 13 * The LOCK_PREFIX macro defined here replaces the LOCK and 14 * LOCK_PREFIX macros used everywhere in the source tree. 15 * 16 * SMP alternatives use the same data structures as the other 17 * alternatives and the X86_FEATURE_UP flag to indicate the case of a 18 * UP system running a SMP kernel. The existing apply_alternatives() 19 * works fine for patching a SMP kernel for UP. 20 * 21 * The SMP alternative tables can be kept after boot and contain both 22 * UP and SMP versions of the instructions to allow switching back to 23 * SMP at runtime, when hotplugging in a new CPU, which is especially 24 * useful in virtualized environments. 25 * 26 * The very common lock prefix is handled as special case in a 27 * separate table which is a pure address list without replacement ptr 28 * and size information. That keeps the table sizes small. 29 */ 30 31 #ifdef CONFIG_SMP 32 #define LOCK_PREFIX_HERE \ 33 ".pushsection .smp_locks,\"a\"\n" \ 34 ".balign 4\n" \ 35 ".long 671f - .\n" /* offset */ \ 36 ".popsection\n" \ 37 "671:" 38 39 #define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; " 40 41 #else /* ! CONFIG_SMP */ 42 #define LOCK_PREFIX_HERE "" 43 #define LOCK_PREFIX "" 44 #endif 45 46 struct alt_instr { 47 s32 instr_offset; /* original instruction */ 48 s32 repl_offset; /* offset to replacement instruction */ 49 u16 cpuid; /* cpuid bit set for replacement */ 50 u8 instrlen; /* length of original instruction */ 51 u8 replacementlen; /* length of new instruction, <= instrlen */ 52 }; 53 54 extern void alternative_instructions(void); 55 extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); 56 57 struct module; 58 59 #ifdef CONFIG_SMP 60 extern void alternatives_smp_module_add(struct module *mod, char *name, 61 void *locks, void *locks_end, 62 void *text, void *text_end); 63 extern void alternatives_smp_module_del(struct module *mod); 64 extern void alternatives_enable_smp(void); 65 extern int alternatives_text_reserved(void *start, void *end); 66 extern bool skip_smp_alternatives; 67 #else 68 static inline void alternatives_smp_module_add(struct module *mod, char *name, 69 void *locks, void *locks_end, 70 void *text, void *text_end) {} 71 static inline void alternatives_smp_module_del(struct module *mod) {} 72 static inline void alternatives_enable_smp(void) {} 73 static inline int alternatives_text_reserved(void *start, void *end) 74 { 75 return 0; 76 } 77 #endif /* CONFIG_SMP */ 78 79 #define OLDINSTR(oldinstr) "661:\n\t" oldinstr "\n662:\n" 80 81 #define b_replacement(number) "663"#number 82 #define e_replacement(number) "664"#number 83 84 #define alt_slen "662b-661b" 85 #define alt_rlen(number) e_replacement(number)"f-"b_replacement(number)"f" 86 87 #define ALTINSTR_ENTRY(feature, number) \ 88 " .long 661b - .\n" /* label */ \ 89 " .long " b_replacement(number)"f - .\n" /* new instruction */ \ 90 " .word " __stringify(feature) "\n" /* feature bit */ \ 91 " .byte " alt_slen "\n" /* source len */ \ 92 " .byte " alt_rlen(number) "\n" /* replacement len */ 93 94 #define DISCARD_ENTRY(number) /* rlen <= slen */ \ 95 " .byte 0xff + (" alt_rlen(number) ") - (" alt_slen ")\n" 96 97 #define ALTINSTR_REPLACEMENT(newinstr, feature, number) /* replacement */ \ 98 b_replacement(number)":\n\t" newinstr "\n" e_replacement(number) ":\n\t" 99 100 /* alternative assembly primitive: */ 101 #define ALTERNATIVE(oldinstr, newinstr, feature) \ 102 OLDINSTR(oldinstr) \ 103 ".pushsection .altinstructions,\"a\"\n" \ 104 ALTINSTR_ENTRY(feature, 1) \ 105 ".popsection\n" \ 106 ".pushsection .discard,\"aw\",@progbits\n" \ 107 DISCARD_ENTRY(1) \ 108 ".popsection\n" \ 109 ".pushsection .altinstr_replacement, \"ax\"\n" \ 110 ALTINSTR_REPLACEMENT(newinstr, feature, 1) \ 111 ".popsection" 112 113 #define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\ 114 OLDINSTR(oldinstr) \ 115 ".pushsection .altinstructions,\"a\"\n" \ 116 ALTINSTR_ENTRY(feature1, 1) \ 117 ALTINSTR_ENTRY(feature2, 2) \ 118 ".popsection\n" \ 119 ".pushsection .discard,\"aw\",@progbits\n" \ 120 DISCARD_ENTRY(1) \ 121 DISCARD_ENTRY(2) \ 122 ".popsection\n" \ 123 ".pushsection .altinstr_replacement, \"ax\"\n" \ 124 ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \ 125 ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \ 126 ".popsection" 127 128 /* 129 * This must be included *after* the definition of ALTERNATIVE due to 130 * <asm/arch_hweight.h> 131 */ 132 #include <asm/cpufeature.h> 133 134 /* 135 * Alternative instructions for different CPU types or capabilities. 136 * 137 * This allows to use optimized instructions even on generic binary 138 * kernels. 139 * 140 * length of oldinstr must be longer or equal the length of newinstr 141 * It can be padded with nops as needed. 142 * 143 * For non barrier like inlines please define new variants 144 * without volatile and memory clobber. 145 */ 146 #define alternative(oldinstr, newinstr, feature) \ 147 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory") 148 149 /* 150 * Alternative inline assembly with input. 151 * 152 * Pecularities: 153 * No memory clobber here. 154 * Argument numbers start with 1. 155 * Best is to use constraints that are fixed size (like (%1) ... "r") 156 * If you use variable sized constraints like "m" or "g" in the 157 * replacement make sure to pad to the worst case length. 158 * Leaving an unused argument 0 to keep API compatibility. 159 */ 160 #define alternative_input(oldinstr, newinstr, feature, input...) \ 161 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ 162 : : "i" (0), ## input) 163 164 /* Like alternative_input, but with a single output argument */ 165 #define alternative_io(oldinstr, newinstr, feature, output, input...) \ 166 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ 167 : output : "i" (0), ## input) 168 169 /* Like alternative_io, but for replacing a direct call with another one. */ 170 #define alternative_call(oldfunc, newfunc, feature, output, input...) \ 171 asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \ 172 : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input) 173 174 /* 175 * Like alternative_call, but there are two features and respective functions. 176 * If CPU has feature2, function2 is used. 177 * Otherwise, if CPU has feature1, function1 is used. 178 * Otherwise, old function is used. 179 */ 180 #define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2, \ 181 output, input...) \ 182 asm volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\ 183 "call %P[new2]", feature2) \ 184 : output : [old] "i" (oldfunc), [new1] "i" (newfunc1), \ 185 [new2] "i" (newfunc2), ## input) 186 187 /* 188 * use this macro(s) if you need more than one output parameter 189 * in alternative_io 190 */ 191 #define ASM_OUTPUT2(a...) a 192 193 /* 194 * use this macro if you need clobbers but no inputs in 195 * alternative_{input,io,call}() 196 */ 197 #define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr 198 199 struct paravirt_patch_site; 200 #ifdef CONFIG_PARAVIRT 201 void apply_paravirt(struct paravirt_patch_site *start, 202 struct paravirt_patch_site *end); 203 #else 204 static inline void apply_paravirt(struct paravirt_patch_site *start, 205 struct paravirt_patch_site *end) 206 {} 207 #define __parainstructions NULL 208 #define __parainstructions_end NULL 209 #endif 210 211 extern void *text_poke_early(void *addr, const void *opcode, size_t len); 212 213 /* 214 * Clear and restore the kernel write-protection flag on the local CPU. 215 * Allows the kernel to edit read-only pages. 216 * Side-effect: any interrupt handler running between save and restore will have 217 * the ability to write to read-only pages. 218 * 219 * Warning: 220 * Code patching in the UP case is safe if NMIs and MCE handlers are stopped and 221 * no thread can be preempted in the instructions being modified (no iret to an 222 * invalid instruction possible) or if the instructions are changed from a 223 * consistent state to another consistent state atomically. 224 * On the local CPU you need to be protected again NMI or MCE handlers seeing an 225 * inconsistent instruction while you patch. 226 */ 227 extern void *text_poke(void *addr, const void *opcode, size_t len); 228 extern int poke_int3_handler(struct pt_regs *regs); 229 extern void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler); 230 231 #endif /* _ASM_X86_ALTERNATIVE_H */ 232