1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_STATIC_CALL_H 3 #define _LINUX_STATIC_CALL_H 4 5 /* 6 * Static call support 7 * 8 * Static calls use code patching to hard-code function pointers into direct 9 * branch instructions. They give the flexibility of function pointers, but 10 * with improved performance. This is especially important for cases where 11 * retpolines would otherwise be used, as retpolines can significantly impact 12 * performance. 13 * 14 * 15 * API overview: 16 * 17 * DECLARE_STATIC_CALL(name, func); 18 * DEFINE_STATIC_CALL(name, func); 19 * DEFINE_STATIC_CALL_NULL(name, typename); 20 * static_call(name)(args...); 21 * static_call_cond(name)(args...); 22 * static_call_update(name, func); 23 * 24 * Usage example: 25 * 26 * # Start with the following functions (with identical prototypes): 27 * int func_a(int arg1, int arg2); 28 * int func_b(int arg1, int arg2); 29 * 30 * # Define a 'my_name' reference, associated with func_a() by default 31 * DEFINE_STATIC_CALL(my_name, func_a); 32 * 33 * # Call func_a() 34 * static_call(my_name)(arg1, arg2); 35 * 36 * # Update 'my_name' to point to func_b() 37 * static_call_update(my_name, &func_b); 38 * 39 * # Call func_b() 40 * static_call(my_name)(arg1, arg2); 41 * 42 * 43 * Implementation details: 44 * 45 * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL). 46 * Otherwise basic indirect calls are used (with function pointers). 47 * 48 * Each static_call() site calls into a trampoline associated with the name. 49 * The trampoline has a direct branch to the default function. Updates to a 50 * name will modify the trampoline's branch destination. 51 * 52 * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites 53 * themselves will be patched at runtime to call the functions directly, 54 * rather than calling through the trampoline. This requires objtool or a 55 * compiler plugin to detect all the static_call() sites and annotate them 56 * in the .static_call_sites section. 57 * 58 * 59 * Notes on NULL function pointers: 60 * 61 * Static_call()s support NULL functions, with many of the caveats that 62 * regular function pointers have. 63 * 64 * Clearly calling a NULL function pointer is 'BAD', so too for 65 * static_call()s (although when HAVE_STATIC_CALL it might not be immediately 66 * fatal). A NULL static_call can be the result of: 67 * 68 * DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int)); 69 * 70 * which is equivalent to declaring a NULL function pointer with just a 71 * typename: 72 * 73 * void (*my_func_ptr)(int arg1) = NULL; 74 * 75 * or using static_call_update() with a NULL function. In both cases the 76 * HAVE_STATIC_CALL implementation will patch the trampoline with a RET 77 * instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE 78 * architectures can patch the trampoline call to a NOP. 79 * 80 * In all cases, any argument evaluation is unconditional. Unlike a regular 81 * conditional function pointer call: 82 * 83 * if (my_func_ptr) 84 * my_func_ptr(arg1) 85 * 86 * where the argument evaludation also depends on the pointer value. 87 * 88 * When calling a static_call that can be NULL, use: 89 * 90 * static_call_cond(name)(arg1); 91 * 92 * which will include the required value tests to avoid NULL-pointer 93 * dereferences. 94 */ 95 96 #include <linux/types.h> 97 #include <linux/cpu.h> 98 #include <linux/static_call_types.h> 99 100 #ifdef CONFIG_HAVE_STATIC_CALL 101 #include <asm/static_call.h> 102 103 /* 104 * Either @site or @tramp can be NULL. 105 */ 106 extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail); 107 108 #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name) 109 110 /* 111 * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from 112 * the symbol table so that objtool can reference it when it generates the 113 * .static_call_sites section. 114 */ 115 #define __static_call(name) \ 116 ({ \ 117 __ADDRESSABLE(STATIC_CALL_KEY(name)); \ 118 &STATIC_CALL_TRAMP(name); \ 119 }) 120 121 #else 122 #define STATIC_CALL_TRAMP_ADDR(name) NULL 123 #endif 124 125 126 #define DECLARE_STATIC_CALL(name, func) \ 127 extern struct static_call_key STATIC_CALL_KEY(name); \ 128 extern typeof(func) STATIC_CALL_TRAMP(name); 129 130 #define static_call_update(name, func) \ 131 ({ \ 132 BUILD_BUG_ON(!__same_type(*(func), STATIC_CALL_TRAMP(name))); \ 133 __static_call_update(&STATIC_CALL_KEY(name), \ 134 STATIC_CALL_TRAMP_ADDR(name), func); \ 135 }) 136 137 #ifdef CONFIG_HAVE_STATIC_CALL_INLINE 138 139 extern int __init static_call_init(void); 140 141 struct static_call_mod { 142 struct static_call_mod *next; 143 struct module *mod; /* for vmlinux, mod == NULL */ 144 struct static_call_site *sites; 145 }; 146 147 struct static_call_key { 148 void *func; 149 union { 150 /* bit 0: 0 = mods, 1 = sites */ 151 unsigned long type; 152 struct static_call_mod *mods; 153 struct static_call_site *sites; 154 }; 155 }; 156 157 extern void __static_call_update(struct static_call_key *key, void *tramp, void *func); 158 extern int static_call_mod_init(struct module *mod); 159 extern int static_call_text_reserved(void *start, void *end); 160 161 #define DEFINE_STATIC_CALL(name, _func) \ 162 DECLARE_STATIC_CALL(name, _func); \ 163 struct static_call_key STATIC_CALL_KEY(name) = { \ 164 .func = _func, \ 165 .type = 1, \ 166 }; \ 167 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) 168 169 #define DEFINE_STATIC_CALL_NULL(name, _func) \ 170 DECLARE_STATIC_CALL(name, _func); \ 171 struct static_call_key STATIC_CALL_KEY(name) = { \ 172 .func = NULL, \ 173 .type = 1, \ 174 }; \ 175 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) 176 177 #define static_call(name) __static_call(name) 178 #define static_call_cond(name) (void)__static_call(name) 179 180 #define EXPORT_STATIC_CALL(name) \ 181 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ 182 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) 183 184 #define EXPORT_STATIC_CALL_GPL(name) \ 185 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ 186 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) 187 188 #elif defined(CONFIG_HAVE_STATIC_CALL) 189 190 static inline int static_call_init(void) { return 0; } 191 192 struct static_call_key { 193 void *func; 194 }; 195 196 #define DEFINE_STATIC_CALL(name, _func) \ 197 DECLARE_STATIC_CALL(name, _func); \ 198 struct static_call_key STATIC_CALL_KEY(name) = { \ 199 .func = _func, \ 200 }; \ 201 ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func) 202 203 #define DEFINE_STATIC_CALL_NULL(name, _func) \ 204 DECLARE_STATIC_CALL(name, _func); \ 205 struct static_call_key STATIC_CALL_KEY(name) = { \ 206 .func = NULL, \ 207 }; \ 208 ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) 209 210 #define static_call(name) __static_call(name) 211 #define static_call_cond(name) (void)__static_call(name) 212 213 static inline 214 void __static_call_update(struct static_call_key *key, void *tramp, void *func) 215 { 216 cpus_read_lock(); 217 WRITE_ONCE(key->func, func); 218 arch_static_call_transform(NULL, tramp, func, false); 219 cpus_read_unlock(); 220 } 221 222 static inline int static_call_text_reserved(void *start, void *end) 223 { 224 return 0; 225 } 226 227 #define EXPORT_STATIC_CALL(name) \ 228 EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ 229 EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) 230 231 #define EXPORT_STATIC_CALL_GPL(name) \ 232 EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ 233 EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) 234 235 #else /* Generic implementation */ 236 237 static inline int static_call_init(void) { return 0; } 238 239 struct static_call_key { 240 void *func; 241 }; 242 243 #define DEFINE_STATIC_CALL(name, _func) \ 244 DECLARE_STATIC_CALL(name, _func); \ 245 struct static_call_key STATIC_CALL_KEY(name) = { \ 246 .func = _func, \ 247 } 248 249 #define DEFINE_STATIC_CALL_NULL(name, _func) \ 250 DECLARE_STATIC_CALL(name, _func); \ 251 struct static_call_key STATIC_CALL_KEY(name) = { \ 252 .func = NULL, \ 253 } 254 255 #define static_call(name) \ 256 ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func)) 257 258 static inline void __static_call_nop(void) { } 259 260 /* 261 * This horrific hack takes care of two things: 262 * 263 * - it ensures the compiler will only load the function pointer ONCE, 264 * which avoids a reload race. 265 * 266 * - it ensures the argument evaluation is unconditional, similar 267 * to the HAVE_STATIC_CALL variant. 268 * 269 * Sadly current GCC/Clang (10 for both) do not optimize this properly 270 * and will emit an indirect call for the NULL case :-( 271 */ 272 #define __static_call_cond(name) \ 273 ({ \ 274 void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \ 275 if (!func) \ 276 func = &__static_call_nop; \ 277 (typeof(STATIC_CALL_TRAMP(name))*)func; \ 278 }) 279 280 #define static_call_cond(name) (void)__static_call_cond(name) 281 282 static inline 283 void __static_call_update(struct static_call_key *key, void *tramp, void *func) 284 { 285 WRITE_ONCE(key->func, func); 286 } 287 288 static inline int static_call_text_reserved(void *start, void *end) 289 { 290 return 0; 291 } 292 293 #define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name)) 294 #define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)) 295 296 #endif /* CONFIG_HAVE_STATIC_CALL */ 297 298 #endif /* _LINUX_STATIC_CALL_H */ 299