1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #define pr_fmt(fmt) "callthunks: " fmt
4
5 #include <linux/debugfs.h>
6 #include <linux/kallsyms.h>
7 #include <linux/memory.h>
8 #include <linux/moduleloader.h>
9 #include <linux/static_call.h>
10
11 #include <asm/alternative.h>
12 #include <asm/asm-offsets.h>
13 #include <asm/cpu.h>
14 #include <asm/ftrace.h>
15 #include <asm/insn.h>
16 #include <asm/kexec.h>
17 #include <asm/nospec-branch.h>
18 #include <asm/paravirt.h>
19 #include <asm/sections.h>
20 #include <asm/switch_to.h>
21 #include <asm/sync_core.h>
22 #include <asm/text-patching.h>
23 #include <asm/xen/hypercall.h>
24
25 static int __initdata_or_module debug_callthunks;
26
27 #define prdbg(fmt, args...) \
28 do { \
29 if (debug_callthunks) \
30 printk(KERN_DEBUG pr_fmt(fmt), ##args); \
31 } while(0)
32
debug_thunks(char * str)33 static int __init debug_thunks(char *str)
34 {
35 debug_callthunks = 1;
36 return 1;
37 }
38 __setup("debug-callthunks", debug_thunks);
39
40 #ifdef CONFIG_CALL_THUNKS_DEBUG
41 DEFINE_PER_CPU(u64, __x86_call_count);
42 DEFINE_PER_CPU(u64, __x86_ret_count);
43 DEFINE_PER_CPU(u64, __x86_stuffs_count);
44 DEFINE_PER_CPU(u64, __x86_ctxsw_count);
45 EXPORT_SYMBOL_GPL(__x86_ctxsw_count);
46 EXPORT_SYMBOL_GPL(__x86_call_count);
47 #endif
48
49 extern s32 __call_sites[], __call_sites_end[];
50
51 struct thunk_desc {
52 void *template;
53 unsigned int template_size;
54 };
55
56 struct core_text {
57 unsigned long base;
58 unsigned long end;
59 const char *name;
60 };
61
62 static bool thunks_initialized __ro_after_init;
63
64 static const struct core_text builtin_coretext = {
65 .base = (unsigned long)_text,
66 .end = (unsigned long)_etext,
67 .name = "builtin",
68 };
69
70 asm (
71 ".pushsection .rodata \n"
72 ".global skl_call_thunk_template \n"
73 "skl_call_thunk_template: \n"
74 __stringify(INCREMENT_CALL_DEPTH)" \n"
75 ".global skl_call_thunk_tail \n"
76 "skl_call_thunk_tail: \n"
77 ".popsection \n"
78 );
79
80 extern u8 skl_call_thunk_template[];
81 extern u8 skl_call_thunk_tail[];
82
83 #define SKL_TMPL_SIZE \
84 ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template))
85
86 extern void error_entry(void);
87 extern void xen_error_entry(void);
88 extern void paranoid_entry(void);
89
within_coretext(const struct core_text * ct,void * addr)90 static inline bool within_coretext(const struct core_text *ct, void *addr)
91 {
92 unsigned long p = (unsigned long)addr;
93
94 return ct->base <= p && p < ct->end;
95 }
96
within_module_coretext(void * addr)97 static inline bool within_module_coretext(void *addr)
98 {
99 bool ret = false;
100
101 #ifdef CONFIG_MODULES
102 struct module *mod;
103
104 preempt_disable();
105 mod = __module_address((unsigned long)addr);
106 if (mod && within_module_core((unsigned long)addr, mod))
107 ret = true;
108 preempt_enable();
109 #endif
110 return ret;
111 }
112
is_coretext(const struct core_text * ct,void * addr)113 static bool is_coretext(const struct core_text *ct, void *addr)
114 {
115 if (ct && within_coretext(ct, addr))
116 return true;
117 if (within_coretext(&builtin_coretext, addr))
118 return true;
119 return within_module_coretext(addr);
120 }
121
skip_addr(void * dest)122 static bool skip_addr(void *dest)
123 {
124 if (dest == error_entry)
125 return true;
126 if (dest == paranoid_entry)
127 return true;
128 if (dest == xen_error_entry)
129 return true;
130 /* Does FILL_RSB... */
131 if (dest == __switch_to_asm)
132 return true;
133 /* Accounts directly */
134 if (dest == ret_from_fork)
135 return true;
136 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT)
137 if (dest == soft_restart_cpu)
138 return true;
139 #endif
140 #ifdef CONFIG_FUNCTION_TRACER
141 if (dest == __fentry__)
142 return true;
143 #endif
144 #ifdef CONFIG_KEXEC_CORE
145 if (dest >= (void *)relocate_kernel &&
146 dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
147 return true;
148 #endif
149 #ifdef CONFIG_XEN
150 if (dest >= (void *)hypercall_page &&
151 dest < (void*)hypercall_page + PAGE_SIZE)
152 return true;
153 #endif
154 return false;
155 }
156
call_get_dest(void * addr)157 static __init_or_module void *call_get_dest(void *addr)
158 {
159 struct insn insn;
160 void *dest;
161 int ret;
162
163 ret = insn_decode_kernel(&insn, addr);
164 if (ret)
165 return ERR_PTR(ret);
166
167 /* Patched out call? */
168 if (insn.opcode.bytes[0] != CALL_INSN_OPCODE)
169 return NULL;
170
171 dest = addr + insn.length + insn.immediate.value;
172 if (skip_addr(dest))
173 return NULL;
174 return dest;
175 }
176
177 static const u8 nops[] = {
178 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
179 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
180 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
181 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
182 };
183
patch_dest(void * dest,bool direct)184 static void *patch_dest(void *dest, bool direct)
185 {
186 unsigned int tsize = SKL_TMPL_SIZE;
187 u8 *pad = dest - tsize;
188
189 /* Already patched? */
190 if (!bcmp(pad, skl_call_thunk_template, tsize))
191 return pad;
192
193 /* Ensure there are nops */
194 if (bcmp(pad, nops, tsize)) {
195 pr_warn_once("Invalid padding area for %pS\n", dest);
196 return NULL;
197 }
198
199 if (direct)
200 memcpy(pad, skl_call_thunk_template, tsize);
201 else
202 text_poke_copy_locked(pad, skl_call_thunk_template, tsize, true);
203 return pad;
204 }
205
patch_call(void * addr,const struct core_text * ct)206 static __init_or_module void patch_call(void *addr, const struct core_text *ct)
207 {
208 void *pad, *dest;
209 u8 bytes[8];
210
211 if (!within_coretext(ct, addr))
212 return;
213
214 dest = call_get_dest(addr);
215 if (!dest || WARN_ON_ONCE(IS_ERR(dest)))
216 return;
217
218 if (!is_coretext(ct, dest))
219 return;
220
221 pad = patch_dest(dest, within_coretext(ct, dest));
222 if (!pad)
223 return;
224
225 prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr,
226 dest, dest, pad);
227 __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE);
228 text_poke_early(addr, bytes, CALL_INSN_SIZE);
229 }
230
231 static __init_or_module void
patch_call_sites(s32 * start,s32 * end,const struct core_text * ct)232 patch_call_sites(s32 *start, s32 *end, const struct core_text *ct)
233 {
234 s32 *s;
235
236 for (s = start; s < end; s++)
237 patch_call((void *)s + *s, ct);
238 }
239
240 static __init_or_module void
patch_paravirt_call_sites(struct paravirt_patch_site * start,struct paravirt_patch_site * end,const struct core_text * ct)241 patch_paravirt_call_sites(struct paravirt_patch_site *start,
242 struct paravirt_patch_site *end,
243 const struct core_text *ct)
244 {
245 struct paravirt_patch_site *p;
246
247 for (p = start; p < end; p++)
248 patch_call(p->instr, ct);
249 }
250
251 static __init_or_module void
callthunks_setup(struct callthunk_sites * cs,const struct core_text * ct)252 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)
253 {
254 prdbg("Patching call sites %s\n", ct->name);
255 patch_call_sites(cs->call_start, cs->call_end, ct);
256 patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct);
257 prdbg("Patching call sites done%s\n", ct->name);
258 }
259
callthunks_patch_builtin_calls(void)260 void __init callthunks_patch_builtin_calls(void)
261 {
262 struct callthunk_sites cs = {
263 .call_start = __call_sites,
264 .call_end = __call_sites_end,
265 .pv_start = __parainstructions,
266 .pv_end = __parainstructions_end
267 };
268
269 if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
270 return;
271
272 pr_info("Setting up call depth tracking\n");
273 mutex_lock(&text_mutex);
274 callthunks_setup(&cs, &builtin_coretext);
275 thunks_initialized = true;
276 mutex_unlock(&text_mutex);
277 }
278
callthunks_translate_call_dest(void * dest)279 void *callthunks_translate_call_dest(void *dest)
280 {
281 void *target;
282
283 lockdep_assert_held(&text_mutex);
284
285 if (!thunks_initialized || skip_addr(dest))
286 return dest;
287
288 if (!is_coretext(NULL, dest))
289 return dest;
290
291 target = patch_dest(dest, false);
292 return target ? : dest;
293 }
294
295 #ifdef CONFIG_BPF_JIT
is_callthunk(void * addr)296 static bool is_callthunk(void *addr)
297 {
298 unsigned int tmpl_size = SKL_TMPL_SIZE;
299 void *tmpl = skl_call_thunk_template;
300 unsigned long dest;
301
302 dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT);
303 if (!thunks_initialized || skip_addr((void *)dest))
304 return false;
305
306 return !bcmp((void *)(dest - tmpl_size), tmpl, tmpl_size);
307 }
308
x86_call_depth_emit_accounting(u8 ** pprog,void * func)309 int x86_call_depth_emit_accounting(u8 **pprog, void *func)
310 {
311 unsigned int tmpl_size = SKL_TMPL_SIZE;
312 void *tmpl = skl_call_thunk_template;
313
314 if (!thunks_initialized)
315 return 0;
316
317 /* Is function call target a thunk? */
318 if (func && is_callthunk(func))
319 return 0;
320
321 memcpy(*pprog, tmpl, tmpl_size);
322 *pprog += tmpl_size;
323 return tmpl_size;
324 }
325 #endif
326
327 #ifdef CONFIG_MODULES
callthunks_patch_module_calls(struct callthunk_sites * cs,struct module * mod)328 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
329 struct module *mod)
330 {
331 struct core_text ct = {
332 .base = (unsigned long)mod->mem[MOD_TEXT].base,
333 .end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size,
334 .name = mod->name,
335 };
336
337 if (!thunks_initialized)
338 return;
339
340 mutex_lock(&text_mutex);
341 callthunks_setup(cs, &ct);
342 mutex_unlock(&text_mutex);
343 }
344 #endif /* CONFIG_MODULES */
345
346 #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
callthunks_debug_show(struct seq_file * m,void * p)347 static int callthunks_debug_show(struct seq_file *m, void *p)
348 {
349 unsigned long cpu = (unsigned long)m->private;
350
351 seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,",
352 per_cpu(__x86_call_count, cpu),
353 per_cpu(__x86_ret_count, cpu),
354 per_cpu(__x86_stuffs_count, cpu),
355 per_cpu(__x86_ctxsw_count, cpu));
356 return 0;
357 }
358
callthunks_debug_open(struct inode * inode,struct file * file)359 static int callthunks_debug_open(struct inode *inode, struct file *file)
360 {
361 return single_open(file, callthunks_debug_show, inode->i_private);
362 }
363
364 static const struct file_operations dfs_ops = {
365 .open = callthunks_debug_open,
366 .read = seq_read,
367 .llseek = seq_lseek,
368 .release = single_release,
369 };
370
callthunks_debugfs_init(void)371 static int __init callthunks_debugfs_init(void)
372 {
373 struct dentry *dir;
374 unsigned long cpu;
375
376 dir = debugfs_create_dir("callthunks", NULL);
377 for_each_possible_cpu(cpu) {
378 void *arg = (void *)cpu;
379 char name [10];
380
381 sprintf(name, "cpu%lu", cpu);
382 debugfs_create_file(name, 0644, dir, arg, &dfs_ops);
383 }
384 return 0;
385 }
386 __initcall(callthunks_debugfs_init);
387 #endif
388