1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #define pr_fmt(fmt) "callthunks: " fmt
4
5 #include <linux/debugfs.h>
6 #include <linux/kallsyms.h>
7 #include <linux/memory.h>
8 #include <linux/moduleloader.h>
9 #include <linux/static_call.h>
10
11 #include <asm/alternative.h>
12 #include <asm/asm-offsets.h>
13 #include <asm/cpu.h>
14 #include <asm/ftrace.h>
15 #include <asm/insn.h>
16 #include <asm/kexec.h>
17 #include <asm/nospec-branch.h>
18 #include <asm/paravirt.h>
19 #include <asm/sections.h>
20 #include <asm/switch_to.h>
21 #include <asm/sync_core.h>
22 #include <asm/text-patching.h>
23 #include <asm/xen/hypercall.h>
24
25 static int __initdata_or_module debug_callthunks;
26
27 #define prdbg(fmt, args...) \
28 do { \
29 if (debug_callthunks) \
30 printk(KERN_DEBUG pr_fmt(fmt), ##args); \
31 } while(0)
32
debug_thunks(char * str)33 static int __init debug_thunks(char *str)
34 {
35 debug_callthunks = 1;
36 return 1;
37 }
38 __setup("debug-callthunks", debug_thunks);
39
40 #ifdef CONFIG_CALL_THUNKS_DEBUG
41 DEFINE_PER_CPU(u64, __x86_call_count);
42 DEFINE_PER_CPU(u64, __x86_ret_count);
43 DEFINE_PER_CPU(u64, __x86_stuffs_count);
44 DEFINE_PER_CPU(u64, __x86_ctxsw_count);
45 EXPORT_SYMBOL_GPL(__x86_ctxsw_count);
46 EXPORT_SYMBOL_GPL(__x86_call_count);
47 #endif
48
49 extern s32 __call_sites[], __call_sites_end[];
50
51 struct thunk_desc {
52 void *template;
53 unsigned int template_size;
54 };
55
56 struct core_text {
57 unsigned long base;
58 unsigned long end;
59 const char *name;
60 };
61
62 static bool thunks_initialized __ro_after_init;
63
64 static const struct core_text builtin_coretext = {
65 .base = (unsigned long)_text,
66 .end = (unsigned long)_etext,
67 .name = "builtin",
68 };
69
70 asm (
71 ".pushsection .rodata \n"
72 ".global skl_call_thunk_template \n"
73 "skl_call_thunk_template: \n"
74 __stringify(INCREMENT_CALL_DEPTH)" \n"
75 ".global skl_call_thunk_tail \n"
76 "skl_call_thunk_tail: \n"
77 ".popsection \n"
78 );
79
80 extern u8 skl_call_thunk_template[];
81 extern u8 skl_call_thunk_tail[];
82
83 #define SKL_TMPL_SIZE \
84 ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template))
85
86 extern void error_entry(void);
87 extern void xen_error_entry(void);
88 extern void paranoid_entry(void);
89
within_coretext(const struct core_text * ct,void * addr)90 static inline bool within_coretext(const struct core_text *ct, void *addr)
91 {
92 unsigned long p = (unsigned long)addr;
93
94 return ct->base <= p && p < ct->end;
95 }
96
within_module_coretext(void * addr)97 static inline bool within_module_coretext(void *addr)
98 {
99 bool ret = false;
100
101 #ifdef CONFIG_MODULES
102 struct module *mod;
103
104 preempt_disable();
105 mod = __module_address((unsigned long)addr);
106 if (mod && within_module_core((unsigned long)addr, mod))
107 ret = true;
108 preempt_enable();
109 #endif
110 return ret;
111 }
112
is_coretext(const struct core_text * ct,void * addr)113 static bool is_coretext(const struct core_text *ct, void *addr)
114 {
115 if (ct && within_coretext(ct, addr))
116 return true;
117 if (within_coretext(&builtin_coretext, addr))
118 return true;
119 return within_module_coretext(addr);
120 }
121
skip_addr(void * dest)122 static bool skip_addr(void *dest)
123 {
124 if (dest == error_entry)
125 return true;
126 if (dest == paranoid_entry)
127 return true;
128 if (dest == xen_error_entry)
129 return true;
130 /* Does FILL_RSB... */
131 if (dest == __switch_to_asm)
132 return true;
133 /* Accounts directly */
134 if (dest == ret_from_fork)
135 return true;
136 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT)
137 if (dest == soft_restart_cpu)
138 return true;
139 #endif
140 #ifdef CONFIG_FUNCTION_TRACER
141 if (dest == __fentry__)
142 return true;
143 #endif
144 #ifdef CONFIG_KEXEC_CORE
145 if (dest >= (void *)relocate_kernel &&
146 dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
147 return true;
148 #endif
149 return false;
150 }
151
call_get_dest(void * addr)152 static __init_or_module void *call_get_dest(void *addr)
153 {
154 struct insn insn;
155 void *dest;
156 int ret;
157
158 ret = insn_decode_kernel(&insn, addr);
159 if (ret)
160 return ERR_PTR(ret);
161
162 /* Patched out call? */
163 if (insn.opcode.bytes[0] != CALL_INSN_OPCODE)
164 return NULL;
165
166 dest = addr + insn.length + insn.immediate.value;
167 if (skip_addr(dest))
168 return NULL;
169 return dest;
170 }
171
172 static const u8 nops[] = {
173 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
174 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
175 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
176 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
177 };
178
patch_dest(void * dest,bool direct)179 static void *patch_dest(void *dest, bool direct)
180 {
181 unsigned int tsize = SKL_TMPL_SIZE;
182 u8 *pad = dest - tsize;
183
184 /* Already patched? */
185 if (!bcmp(pad, skl_call_thunk_template, tsize))
186 return pad;
187
188 /* Ensure there are nops */
189 if (bcmp(pad, nops, tsize)) {
190 pr_warn_once("Invalid padding area for %pS\n", dest);
191 return NULL;
192 }
193
194 if (direct)
195 memcpy(pad, skl_call_thunk_template, tsize);
196 else
197 text_poke_copy_locked(pad, skl_call_thunk_template, tsize, true);
198 return pad;
199 }
200
patch_call(void * addr,const struct core_text * ct)201 static __init_or_module void patch_call(void *addr, const struct core_text *ct)
202 {
203 void *pad, *dest;
204 u8 bytes[8];
205
206 if (!within_coretext(ct, addr))
207 return;
208
209 dest = call_get_dest(addr);
210 if (!dest || WARN_ON_ONCE(IS_ERR(dest)))
211 return;
212
213 if (!is_coretext(ct, dest))
214 return;
215
216 pad = patch_dest(dest, within_coretext(ct, dest));
217 if (!pad)
218 return;
219
220 prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr,
221 dest, dest, pad);
222 __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE);
223 text_poke_early(addr, bytes, CALL_INSN_SIZE);
224 }
225
226 static __init_or_module void
patch_call_sites(s32 * start,s32 * end,const struct core_text * ct)227 patch_call_sites(s32 *start, s32 *end, const struct core_text *ct)
228 {
229 s32 *s;
230
231 for (s = start; s < end; s++)
232 patch_call((void *)s + *s, ct);
233 }
234
235 static __init_or_module void
patch_paravirt_call_sites(struct paravirt_patch_site * start,struct paravirt_patch_site * end,const struct core_text * ct)236 patch_paravirt_call_sites(struct paravirt_patch_site *start,
237 struct paravirt_patch_site *end,
238 const struct core_text *ct)
239 {
240 struct paravirt_patch_site *p;
241
242 for (p = start; p < end; p++)
243 patch_call(p->instr, ct);
244 }
245
246 static __init_or_module void
callthunks_setup(struct callthunk_sites * cs,const struct core_text * ct)247 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)
248 {
249 prdbg("Patching call sites %s\n", ct->name);
250 patch_call_sites(cs->call_start, cs->call_end, ct);
251 patch_paravirt_call_sites(cs->pv_start, cs->pv_end, ct);
252 prdbg("Patching call sites done%s\n", ct->name);
253 }
254
callthunks_patch_builtin_calls(void)255 void __init callthunks_patch_builtin_calls(void)
256 {
257 struct callthunk_sites cs = {
258 .call_start = __call_sites,
259 .call_end = __call_sites_end,
260 .pv_start = __parainstructions,
261 .pv_end = __parainstructions_end
262 };
263
264 if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
265 return;
266
267 pr_info("Setting up call depth tracking\n");
268 mutex_lock(&text_mutex);
269 callthunks_setup(&cs, &builtin_coretext);
270 thunks_initialized = true;
271 mutex_unlock(&text_mutex);
272 }
273
callthunks_translate_call_dest(void * dest)274 void *callthunks_translate_call_dest(void *dest)
275 {
276 void *target;
277
278 lockdep_assert_held(&text_mutex);
279
280 if (!thunks_initialized || skip_addr(dest))
281 return dest;
282
283 if (!is_coretext(NULL, dest))
284 return dest;
285
286 target = patch_dest(dest, false);
287 return target ? : dest;
288 }
289
290 #ifdef CONFIG_BPF_JIT
is_callthunk(void * addr)291 static bool is_callthunk(void *addr)
292 {
293 unsigned int tmpl_size = SKL_TMPL_SIZE;
294 void *tmpl = skl_call_thunk_template;
295 unsigned long dest;
296
297 dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT);
298 if (!thunks_initialized || skip_addr((void *)dest))
299 return false;
300
301 return !bcmp((void *)(dest - tmpl_size), tmpl, tmpl_size);
302 }
303
x86_call_depth_emit_accounting(u8 ** pprog,void * func)304 int x86_call_depth_emit_accounting(u8 **pprog, void *func)
305 {
306 unsigned int tmpl_size = SKL_TMPL_SIZE;
307 void *tmpl = skl_call_thunk_template;
308
309 if (!thunks_initialized)
310 return 0;
311
312 /* Is function call target a thunk? */
313 if (func && is_callthunk(func))
314 return 0;
315
316 memcpy(*pprog, tmpl, tmpl_size);
317 *pprog += tmpl_size;
318 return tmpl_size;
319 }
320 #endif
321
322 #ifdef CONFIG_MODULES
callthunks_patch_module_calls(struct callthunk_sites * cs,struct module * mod)323 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
324 struct module *mod)
325 {
326 struct core_text ct = {
327 .base = (unsigned long)mod->mem[MOD_TEXT].base,
328 .end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size,
329 .name = mod->name,
330 };
331
332 if (!thunks_initialized)
333 return;
334
335 mutex_lock(&text_mutex);
336 callthunks_setup(cs, &ct);
337 mutex_unlock(&text_mutex);
338 }
339 #endif /* CONFIG_MODULES */
340
341 #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
callthunks_debug_show(struct seq_file * m,void * p)342 static int callthunks_debug_show(struct seq_file *m, void *p)
343 {
344 unsigned long cpu = (unsigned long)m->private;
345
346 seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,",
347 per_cpu(__x86_call_count, cpu),
348 per_cpu(__x86_ret_count, cpu),
349 per_cpu(__x86_stuffs_count, cpu),
350 per_cpu(__x86_ctxsw_count, cpu));
351 return 0;
352 }
353
callthunks_debug_open(struct inode * inode,struct file * file)354 static int callthunks_debug_open(struct inode *inode, struct file *file)
355 {
356 return single_open(file, callthunks_debug_show, inode->i_private);
357 }
358
359 static const struct file_operations dfs_ops = {
360 .open = callthunks_debug_open,
361 .read = seq_read,
362 .llseek = seq_lseek,
363 .release = single_release,
364 };
365
callthunks_debugfs_init(void)366 static int __init callthunks_debugfs_init(void)
367 {
368 struct dentry *dir;
369 unsigned long cpu;
370
371 dir = debugfs_create_dir("callthunks", NULL);
372 for_each_possible_cpu(cpu) {
373 void *arg = (void *)cpu;
374 char name [10];
375
376 sprintf(name, "cpu%lu", cpu);
377 debugfs_create_file(name, 0644, dir, arg, &dfs_ops);
378 }
379 return 0;
380 }
381 __initcall(callthunks_debugfs_init);
382 #endif
383