1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #define pr_fmt(fmt) "callthunks: " fmt 4 5 #include <linux/debugfs.h> 6 #include <linux/kallsyms.h> 7 #include <linux/memory.h> 8 #include <linux/moduleloader.h> 9 10 #include <asm/alternative.h> 11 #include <asm/asm-offsets.h> 12 #include <asm/cpu.h> 13 #include <asm/ftrace.h> 14 #include <asm/insn.h> 15 #include <asm/kexec.h> 16 #include <asm/nospec-branch.h> 17 #include <asm/paravirt.h> 18 #include <asm/sections.h> 19 #include <asm/switch_to.h> 20 #include <asm/sync_core.h> 21 #include <asm/text-patching.h> 22 #include <asm/xen/hypercall.h> 23 24 static int __initdata_or_module debug_callthunks; 25 26 #define prdbg(fmt, args...) \ 27 do { \ 28 if (debug_callthunks) \ 29 printk(KERN_DEBUG pr_fmt(fmt), ##args); \ 30 } while(0) 31 32 static int __init debug_thunks(char *str) 33 { 34 debug_callthunks = 1; 35 return 1; 36 } 37 __setup("debug-callthunks", debug_thunks); 38 39 #ifdef CONFIG_CALL_THUNKS_DEBUG 40 DEFINE_PER_CPU(u64, __x86_call_count); 41 DEFINE_PER_CPU(u64, __x86_ret_count); 42 DEFINE_PER_CPU(u64, __x86_stuffs_count); 43 DEFINE_PER_CPU(u64, __x86_ctxsw_count); 44 EXPORT_SYMBOL_GPL(__x86_ctxsw_count); 45 EXPORT_SYMBOL_GPL(__x86_call_count); 46 #endif 47 48 extern s32 __call_sites[], __call_sites_end[]; 49 50 struct thunk_desc { 51 void *template; 52 unsigned int template_size; 53 }; 54 55 struct core_text { 56 unsigned long base; 57 unsigned long end; 58 const char *name; 59 }; 60 61 static bool thunks_initialized __ro_after_init; 62 63 static const struct core_text builtin_coretext = { 64 .base = (unsigned long)_text, 65 .end = (unsigned long)_etext, 66 .name = "builtin", 67 }; 68 69 asm ( 70 ".pushsection .rodata \n" 71 ".global skl_call_thunk_template \n" 72 "skl_call_thunk_template: \n" 73 __stringify(INCREMENT_CALL_DEPTH)" \n" 74 ".global skl_call_thunk_tail \n" 75 "skl_call_thunk_tail: \n" 76 ".popsection \n" 77 ); 78 79 extern u8 skl_call_thunk_template[]; 80 extern u8 skl_call_thunk_tail[]; 81 82 #define SKL_TMPL_SIZE \ 83 ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template)) 84 85 extern void error_entry(void); 86 extern void xen_error_entry(void); 87 extern void paranoid_entry(void); 88 89 static inline bool within_coretext(const struct core_text *ct, void *addr) 90 { 91 unsigned long p = (unsigned long)addr; 92 93 return ct->base <= p && p < ct->end; 94 } 95 96 static inline bool within_module_coretext(void *addr) 97 { 98 bool ret = false; 99 100 #ifdef CONFIG_MODULES 101 struct module *mod; 102 103 preempt_disable(); 104 mod = __module_address((unsigned long)addr); 105 if (mod && within_module_core((unsigned long)addr, mod)) 106 ret = true; 107 preempt_enable(); 108 #endif 109 return ret; 110 } 111 112 static bool is_coretext(const struct core_text *ct, void *addr) 113 { 114 if (ct && within_coretext(ct, addr)) 115 return true; 116 if (within_coretext(&builtin_coretext, addr)) 117 return true; 118 return within_module_coretext(addr); 119 } 120 121 static __init_or_module bool skip_addr(void *dest) 122 { 123 if (dest == error_entry) 124 return true; 125 if (dest == paranoid_entry) 126 return true; 127 if (dest == xen_error_entry) 128 return true; 129 /* Does FILL_RSB... */ 130 if (dest == __switch_to_asm) 131 return true; 132 /* Accounts directly */ 133 if (dest == ret_from_fork) 134 return true; 135 #ifdef CONFIG_HOTPLUG_CPU 136 if (dest == start_cpu0) 137 return true; 138 #endif 139 #ifdef CONFIG_FUNCTION_TRACER 140 if (dest == __fentry__) 141 return true; 142 #endif 143 #ifdef CONFIG_KEXEC_CORE 144 if (dest >= (void *)relocate_kernel && 145 dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE) 146 return true; 147 #endif 148 #ifdef CONFIG_XEN 149 if (dest >= (void *)hypercall_page && 150 dest < (void*)hypercall_page + PAGE_SIZE) 151 return true; 152 #endif 153 return false; 154 } 155 156 static __init_or_module void *call_get_dest(void *addr) 157 { 158 struct insn insn; 159 void *dest; 160 int ret; 161 162 ret = insn_decode_kernel(&insn, addr); 163 if (ret) 164 return ERR_PTR(ret); 165 166 /* Patched out call? */ 167 if (insn.opcode.bytes[0] != CALL_INSN_OPCODE) 168 return NULL; 169 170 dest = addr + insn.length + insn.immediate.value; 171 if (skip_addr(dest)) 172 return NULL; 173 return dest; 174 } 175 176 static const u8 nops[] = { 177 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 178 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 179 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 180 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 181 }; 182 183 static __init_or_module void *patch_dest(void *dest, bool direct) 184 { 185 unsigned int tsize = SKL_TMPL_SIZE; 186 u8 *pad = dest - tsize; 187 188 /* Already patched? */ 189 if (!bcmp(pad, skl_call_thunk_template, tsize)) 190 return pad; 191 192 /* Ensure there are nops */ 193 if (bcmp(pad, nops, tsize)) { 194 pr_warn_once("Invalid padding area for %pS\n", dest); 195 return NULL; 196 } 197 198 if (direct) 199 memcpy(pad, skl_call_thunk_template, tsize); 200 else 201 text_poke_copy_locked(pad, skl_call_thunk_template, tsize, true); 202 return pad; 203 } 204 205 static __init_or_module void patch_call(void *addr, const struct core_text *ct) 206 { 207 void *pad, *dest; 208 u8 bytes[8]; 209 210 if (!within_coretext(ct, addr)) 211 return; 212 213 dest = call_get_dest(addr); 214 if (!dest || WARN_ON_ONCE(IS_ERR(dest))) 215 return; 216 217 if (!is_coretext(ct, dest)) 218 return; 219 220 pad = patch_dest(dest, within_coretext(ct, dest)); 221 if (!pad) 222 return; 223 224 prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr, 225 dest, dest, pad); 226 __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE); 227 text_poke_early(addr, bytes, CALL_INSN_SIZE); 228 } 229 230 static __init_or_module void 231 patch_call_sites(s32 *start, s32 *end, const struct core_text *ct) 232 { 233 s32 *s; 234 235 for (s = start; s < end; s++) 236 patch_call((void *)s + *s, ct); 237 } 238 239 static __init_or_module void 240 patch_paravirt_call_sites(struct paravirt_patch_site *start, 241 struct paravirt_patch_site *end, 242 const struct core_text *ct) 243 { 244 struct paravirt_patch_site *p; 245 246 for (p = start; p < end; p++) 247 patch_call(p->instr, ct); 248 } 249 250 static __init_or_module void 251 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct) 252 { 253 prdbg("Patching call sites %s\n", ct->name); 254 patch_call_sites(cs->call_start, cs->call_end, ct); 255 patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct); 256 prdbg("Patching call sites done%s\n", ct->name); 257 } 258 259 void __init callthunks_patch_builtin_calls(void) 260 { 261 struct callthunk_sites cs = { 262 .call_start = __call_sites, 263 .call_end = __call_sites_end, 264 .pv_start = __parainstructions, 265 .pv_end = __parainstructions_end 266 }; 267 268 if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH)) 269 return; 270 271 pr_info("Setting up call depth tracking\n"); 272 mutex_lock(&text_mutex); 273 callthunks_setup(&cs, &builtin_coretext); 274 thunks_initialized = true; 275 mutex_unlock(&text_mutex); 276 } 277 278 #ifdef CONFIG_MODULES 279 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs, 280 struct module *mod) 281 { 282 struct core_text ct = { 283 .base = (unsigned long)mod->core_layout.base, 284 .end = (unsigned long)mod->core_layout.base + mod->core_layout.size, 285 .name = mod->name, 286 }; 287 288 if (!thunks_initialized) 289 return; 290 291 mutex_lock(&text_mutex); 292 callthunks_setup(cs, &ct); 293 mutex_unlock(&text_mutex); 294 } 295 #endif /* CONFIG_MODULES */ 296 297 #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS) 298 static int callthunks_debug_show(struct seq_file *m, void *p) 299 { 300 unsigned long cpu = (unsigned long)m->private; 301 302 seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,", 303 per_cpu(__x86_call_count, cpu), 304 per_cpu(__x86_ret_count, cpu), 305 per_cpu(__x86_stuffs_count, cpu), 306 per_cpu(__x86_ctxsw_count, cpu)); 307 return 0; 308 } 309 310 static int callthunks_debug_open(struct inode *inode, struct file *file) 311 { 312 return single_open(file, callthunks_debug_show, inode->i_private); 313 } 314 315 static const struct file_operations dfs_ops = { 316 .open = callthunks_debug_open, 317 .read = seq_read, 318 .llseek = seq_lseek, 319 .release = single_release, 320 }; 321 322 static int __init callthunks_debugfs_init(void) 323 { 324 struct dentry *dir; 325 unsigned long cpu; 326 327 dir = debugfs_create_dir("callthunks", NULL); 328 for_each_possible_cpu(cpu) { 329 void *arg = (void *)cpu; 330 char name [10]; 331 332 sprintf(name, "cpu%lu", cpu); 333 debugfs_create_file(name, 0644, dir, arg, &dfs_ops); 334 } 335 return 0; 336 } 337 __initcall(callthunks_debugfs_init); 338 #endif 339