1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
247788c58SFrederic Weisbecker #include <trace/syscall.h>
31c569f02SJosh Stone #include <trace/events/syscalls.h>
4f431b634SSteven Rostedt #include <linux/syscalls.h>
55a0e3ad6STejun Heo #include <linux/slab.h>
6ee08c6ecSFrederic Weisbecker #include <linux/kernel.h>
756d82e00SPaul Gortmaker #include <linux/module.h> /* for MODULE_NAME_LEN via KSYM_SYMBOL_LEN */
8fb34a08cSJason Baron #include <linux/ftrace.h>
9cdd6c482SIngo Molnar #include <linux/perf_event.h>
100e242208SHassan Naveed #include <linux/xarray.h>
11ee08c6ecSFrederic Weisbecker #include <asm/syscall.h>
12ee08c6ecSFrederic Weisbecker
13ee08c6ecSFrederic Weisbecker #include "trace_output.h"
14ee08c6ecSFrederic Weisbecker #include "trace.h"
15ee08c6ecSFrederic Weisbecker
165be71b61SFrederic Weisbecker static DEFINE_MUTEX(syscall_trace_lock);
17ee08c6ecSFrederic Weisbecker
182425bcb9SSteven Rostedt (Red Hat) static int syscall_enter_register(struct trace_event_call *event,
19ceec0b6fSJiri Olsa enum trace_reg type, void *data);
202425bcb9SSteven Rostedt (Red Hat) static int syscall_exit_register(struct trace_event_call *event,
21ceec0b6fSJiri Olsa enum trace_reg type, void *data);
222239291aSSteven Rostedt
232e33af02SSteven Rostedt static struct list_head *
syscall_get_enter_fields(struct trace_event_call * call)242425bcb9SSteven Rostedt (Red Hat) syscall_get_enter_fields(struct trace_event_call *call)
252e33af02SSteven Rostedt {
262e33af02SSteven Rostedt struct syscall_metadata *entry = call->data;
272e33af02SSteven Rostedt
282e33af02SSteven Rostedt return &entry->enter_fields;
292e33af02SSteven Rostedt }
302e33af02SSteven Rostedt
313d56e331SSteven Rostedt extern struct syscall_metadata *__start_syscalls_metadata[];
323d56e331SSteven Rostedt extern struct syscall_metadata *__stop_syscalls_metadata[];
33c44fc770SFrederic Weisbecker
340e242208SHassan Naveed static DEFINE_XARRAY(syscalls_metadata_sparse);
35c44fc770SFrederic Weisbecker static struct syscall_metadata **syscalls_metadata;
36c44fc770SFrederic Weisbecker
37b2d55496SIan Munsie #ifndef ARCH_HAS_SYSCALL_MATCH_SYM_NAME
arch_syscall_match_sym_name(const char * sym,const char * name)38b2d55496SIan Munsie static inline bool arch_syscall_match_sym_name(const char *sym, const char *name)
39b2d55496SIan Munsie {
40b2d55496SIan Munsie /*
41b2d55496SIan Munsie * Only compare after the "sys" prefix. Archs that use
42b2d55496SIan Munsie * syscall wrappers may have syscalls symbols aliases prefixed
4336a78e9eSzhangwei(Jovi) * with ".SyS" or ".sys" instead of "sys", leading to an unwanted
44b2d55496SIan Munsie * mismatch.
45b2d55496SIan Munsie */
46b2d55496SIan Munsie return !strcmp(sym + 3, name + 3);
47b2d55496SIan Munsie }
48b2d55496SIan Munsie #endif
49b2d55496SIan Munsie
50f431b634SSteven Rostedt #ifdef ARCH_TRACE_IGNORE_COMPAT_SYSCALLS
51f431b634SSteven Rostedt /*
52f431b634SSteven Rostedt * Some architectures that allow for 32bit applications
53f431b634SSteven Rostedt * to run on a 64bit kernel, do not map the syscalls for
54f431b634SSteven Rostedt * the 32bit tasks the same as they do for 64bit tasks.
55f431b634SSteven Rostedt *
56f431b634SSteven Rostedt * *cough*x86*cough*
57f431b634SSteven Rostedt *
58f431b634SSteven Rostedt * In such a case, instead of reporting the wrong syscalls,
59f431b634SSteven Rostedt * simply ignore them.
60f431b634SSteven Rostedt *
61f431b634SSteven Rostedt * For an arch to ignore the compat syscalls it needs to
62f431b634SSteven Rostedt * define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS as well as
63f431b634SSteven Rostedt * define the function arch_trace_is_compat_syscall() to let
64f431b634SSteven Rostedt * the tracing system know that it should ignore it.
65f431b634SSteven Rostedt */
66f431b634SSteven Rostedt static int
trace_get_syscall_nr(struct task_struct * task,struct pt_regs * regs)67f431b634SSteven Rostedt trace_get_syscall_nr(struct task_struct *task, struct pt_regs *regs)
68f431b634SSteven Rostedt {
69f431b634SSteven Rostedt if (unlikely(arch_trace_is_compat_syscall(regs)))
70f431b634SSteven Rostedt return -1;
71f431b634SSteven Rostedt
72f431b634SSteven Rostedt return syscall_get_nr(task, regs);
73f431b634SSteven Rostedt }
74f431b634SSteven Rostedt #else
75f431b634SSteven Rostedt static inline int
trace_get_syscall_nr(struct task_struct * task,struct pt_regs * regs)76f431b634SSteven Rostedt trace_get_syscall_nr(struct task_struct *task, struct pt_regs *regs)
77f431b634SSteven Rostedt {
78f431b634SSteven Rostedt return syscall_get_nr(task, regs);
79f431b634SSteven Rostedt }
80f431b634SSteven Rostedt #endif /* ARCH_TRACE_IGNORE_COMPAT_SYSCALLS */
81f431b634SSteven Rostedt
823d56e331SSteven Rostedt static __init struct syscall_metadata *
find_syscall_meta(unsigned long syscall)833d56e331SSteven Rostedt find_syscall_meta(unsigned long syscall)
84c44fc770SFrederic Weisbecker {
853d56e331SSteven Rostedt struct syscall_metadata **start;
863d56e331SSteven Rostedt struct syscall_metadata **stop;
87c44fc770SFrederic Weisbecker char str[KSYM_SYMBOL_LEN];
88c44fc770SFrederic Weisbecker
89c44fc770SFrederic Weisbecker
903d56e331SSteven Rostedt start = __start_syscalls_metadata;
913d56e331SSteven Rostedt stop = __stop_syscalls_metadata;
92c44fc770SFrederic Weisbecker kallsyms_lookup(syscall, NULL, NULL, NULL, str);
93c44fc770SFrederic Weisbecker
94ae07f551SIan Munsie if (arch_syscall_match_sym_name(str, "sys_ni_syscall"))
95ae07f551SIan Munsie return NULL;
96ae07f551SIan Munsie
97c44fc770SFrederic Weisbecker for ( ; start < stop; start++) {
98b2d55496SIan Munsie if ((*start)->name && arch_syscall_match_sym_name(str, (*start)->name))
993d56e331SSteven Rostedt return *start;
100c44fc770SFrederic Weisbecker }
101c44fc770SFrederic Weisbecker return NULL;
102c44fc770SFrederic Weisbecker }
103c44fc770SFrederic Weisbecker
syscall_nr_to_meta(int nr)104c44fc770SFrederic Weisbecker static struct syscall_metadata *syscall_nr_to_meta(int nr)
105c44fc770SFrederic Weisbecker {
1060e242208SHassan Naveed if (IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR))
1070e242208SHassan Naveed return xa_load(&syscalls_metadata_sparse, (unsigned long)nr);
1080e242208SHassan Naveed
109c44fc770SFrederic Weisbecker if (!syscalls_metadata || nr >= NR_syscalls || nr < 0)
110c44fc770SFrederic Weisbecker return NULL;
111c44fc770SFrederic Weisbecker
112c44fc770SFrederic Weisbecker return syscalls_metadata[nr];
113c44fc770SFrederic Weisbecker }
114c44fc770SFrederic Weisbecker
get_syscall_name(int syscall)115dbfeaa7aSTom Zanussi const char *get_syscall_name(int syscall)
116dbfeaa7aSTom Zanussi {
117dbfeaa7aSTom Zanussi struct syscall_metadata *entry;
118dbfeaa7aSTom Zanussi
119dbfeaa7aSTom Zanussi entry = syscall_nr_to_meta(syscall);
120dbfeaa7aSTom Zanussi if (!entry)
121dbfeaa7aSTom Zanussi return NULL;
122dbfeaa7aSTom Zanussi
123dbfeaa7aSTom Zanussi return entry->name;
124dbfeaa7aSTom Zanussi }
125dbfeaa7aSTom Zanussi
1266aea49cbSFengguang Wu static enum print_line_t
print_syscall_enter(struct trace_iterator * iter,int flags,struct trace_event * event)127a9a57763SSteven Rostedt print_syscall_enter(struct trace_iterator *iter, int flags,
128a9a57763SSteven Rostedt struct trace_event *event)
129bed1ffcaSFrederic Weisbecker {
130983f938aSSteven Rostedt (Red Hat) struct trace_array *tr = iter->tr;
131bed1ffcaSFrederic Weisbecker struct trace_seq *s = &iter->seq;
132bed1ffcaSFrederic Weisbecker struct trace_entry *ent = iter->ent;
133bed1ffcaSFrederic Weisbecker struct syscall_trace_enter *trace;
134bed1ffcaSFrederic Weisbecker struct syscall_metadata *entry;
135183742f0SSteven Rostedt (Red Hat) int i, syscall;
136bed1ffcaSFrederic Weisbecker
13764c12e04SJason Baron trace = (typeof(trace))ent;
138bed1ffcaSFrederic Weisbecker syscall = trace->nr;
139bed1ffcaSFrederic Weisbecker entry = syscall_nr_to_meta(syscall);
14064c12e04SJason Baron
141bed1ffcaSFrederic Weisbecker if (!entry)
142bed1ffcaSFrederic Weisbecker goto end;
143bed1ffcaSFrederic Weisbecker
14432c0edaeSSteven Rostedt if (entry->enter_event->event.type != ent->type) {
14564c12e04SJason Baron WARN_ON_ONCE(1);
14664c12e04SJason Baron goto end;
14764c12e04SJason Baron }
14864c12e04SJason Baron
149183742f0SSteven Rostedt (Red Hat) trace_seq_printf(s, "%s(", entry->name);
150bed1ffcaSFrederic Weisbecker
151bed1ffcaSFrederic Weisbecker for (i = 0; i < entry->nb_args; i++) {
152183742f0SSteven Rostedt (Red Hat)
153183742f0SSteven Rostedt (Red Hat) if (trace_seq_has_overflowed(s))
154183742f0SSteven Rostedt (Red Hat) goto end;
155183742f0SSteven Rostedt (Red Hat)
156bed1ffcaSFrederic Weisbecker /* parameter types */
157cb1c45fbSJeff Xie if (tr && tr->trace_flags & TRACE_ITER_VERBOSE)
158183742f0SSteven Rostedt (Red Hat) trace_seq_printf(s, "%s ", entry->types[i]);
159183742f0SSteven Rostedt (Red Hat)
160bed1ffcaSFrederic Weisbecker /* parameter values */
161183742f0SSteven Rostedt (Red Hat) trace_seq_printf(s, "%s: %lx%s", entry->args[i],
162bed1ffcaSFrederic Weisbecker trace->args[i],
1634539f077SLi Zefan i == entry->nb_args - 1 ? "" : ", ");
164bed1ffcaSFrederic Weisbecker }
165bed1ffcaSFrederic Weisbecker
166183742f0SSteven Rostedt (Red Hat) trace_seq_putc(s, ')');
167bed1ffcaSFrederic Weisbecker end:
168183742f0SSteven Rostedt (Red Hat) trace_seq_putc(s, '\n');
1694539f077SLi Zefan
170183742f0SSteven Rostedt (Red Hat) return trace_handle_return(s);
171bed1ffcaSFrederic Weisbecker }
172bed1ffcaSFrederic Weisbecker
1736aea49cbSFengguang Wu static enum print_line_t
print_syscall_exit(struct trace_iterator * iter,int flags,struct trace_event * event)174a9a57763SSteven Rostedt print_syscall_exit(struct trace_iterator *iter, int flags,
175a9a57763SSteven Rostedt struct trace_event *event)
176bed1ffcaSFrederic Weisbecker {
177bed1ffcaSFrederic Weisbecker struct trace_seq *s = &iter->seq;
178bed1ffcaSFrederic Weisbecker struct trace_entry *ent = iter->ent;
179bed1ffcaSFrederic Weisbecker struct syscall_trace_exit *trace;
180bed1ffcaSFrederic Weisbecker int syscall;
181bed1ffcaSFrederic Weisbecker struct syscall_metadata *entry;
182bed1ffcaSFrederic Weisbecker
18364c12e04SJason Baron trace = (typeof(trace))ent;
184bed1ffcaSFrederic Weisbecker syscall = trace->nr;
185bed1ffcaSFrederic Weisbecker entry = syscall_nr_to_meta(syscall);
18664c12e04SJason Baron
187bed1ffcaSFrederic Weisbecker if (!entry) {
188146c3442Szhangwei(Jovi) trace_seq_putc(s, '\n');
189183742f0SSteven Rostedt (Red Hat) goto out;
190bed1ffcaSFrederic Weisbecker }
191bed1ffcaSFrederic Weisbecker
19232c0edaeSSteven Rostedt if (entry->exit_event->event.type != ent->type) {
19364c12e04SJason Baron WARN_ON_ONCE(1);
19464c12e04SJason Baron return TRACE_TYPE_UNHANDLED;
19564c12e04SJason Baron }
19664c12e04SJason Baron
197183742f0SSteven Rostedt (Red Hat) trace_seq_printf(s, "%s -> 0x%lx\n", entry->name,
198bed1ffcaSFrederic Weisbecker trace->ret);
199bed1ffcaSFrederic Weisbecker
200183742f0SSteven Rostedt (Red Hat) out:
201183742f0SSteven Rostedt (Red Hat) return trace_handle_return(s);
202bed1ffcaSFrederic Weisbecker }
203bed1ffcaSFrederic Weisbecker
20404ae87a5SPeter Zijlstra #define SYSCALL_FIELD(_type, _name) { \
20504ae87a5SPeter Zijlstra .type = #_type, .name = #_name, \
20604ae87a5SPeter Zijlstra .size = sizeof(_type), .align = __alignof__(_type), \
20704ae87a5SPeter Zijlstra .is_signed = is_signed_type(_type), .filter_type = FILTER_OTHER }
208e6971969SLi Zefan
2093ddc77f6SLi Zefan static int __init
__set_enter_print_fmt(struct syscall_metadata * entry,char * buf,int len)2103ddc77f6SLi Zefan __set_enter_print_fmt(struct syscall_metadata *entry, char *buf, int len)
21150307a45SLai Jiangshan {
21250307a45SLai Jiangshan int i;
21350307a45SLai Jiangshan int pos = 0;
21450307a45SLai Jiangshan
21550307a45SLai Jiangshan /* When len=0, we just calculate the needed length */
21650307a45SLai Jiangshan #define LEN_OR_ZERO (len ? len - pos : 0)
21750307a45SLai Jiangshan
21850307a45SLai Jiangshan pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
21950307a45SLai Jiangshan for (i = 0; i < entry->nb_args; i++) {
22050307a45SLai Jiangshan pos += snprintf(buf + pos, LEN_OR_ZERO, "%s: 0x%%0%zulx%s",
22150307a45SLai Jiangshan entry->args[i], sizeof(unsigned long),
22250307a45SLai Jiangshan i == entry->nb_args - 1 ? "" : ", ");
22350307a45SLai Jiangshan }
22450307a45SLai Jiangshan pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
22550307a45SLai Jiangshan
22650307a45SLai Jiangshan for (i = 0; i < entry->nb_args; i++) {
22750307a45SLai Jiangshan pos += snprintf(buf + pos, LEN_OR_ZERO,
22850307a45SLai Jiangshan ", ((unsigned long)(REC->%s))", entry->args[i]);
22950307a45SLai Jiangshan }
23050307a45SLai Jiangshan
23150307a45SLai Jiangshan #undef LEN_OR_ZERO
23250307a45SLai Jiangshan
23350307a45SLai Jiangshan /* return the length of print_fmt */
23450307a45SLai Jiangshan return pos;
23550307a45SLai Jiangshan }
23650307a45SLai Jiangshan
set_syscall_print_fmt(struct trace_event_call * call)2372425bcb9SSteven Rostedt (Red Hat) static int __init set_syscall_print_fmt(struct trace_event_call *call)
23850307a45SLai Jiangshan {
23950307a45SLai Jiangshan char *print_fmt;
24050307a45SLai Jiangshan int len;
24150307a45SLai Jiangshan struct syscall_metadata *entry = call->data;
24250307a45SLai Jiangshan
24350307a45SLai Jiangshan if (entry->enter_event != call) {
24450307a45SLai Jiangshan call->print_fmt = "\"0x%lx\", REC->ret";
24550307a45SLai Jiangshan return 0;
24650307a45SLai Jiangshan }
24750307a45SLai Jiangshan
24850307a45SLai Jiangshan /* First: called with 0 length to calculate the needed length */
24950307a45SLai Jiangshan len = __set_enter_print_fmt(entry, NULL, 0);
25050307a45SLai Jiangshan
25150307a45SLai Jiangshan print_fmt = kmalloc(len + 1, GFP_KERNEL);
25250307a45SLai Jiangshan if (!print_fmt)
25350307a45SLai Jiangshan return -ENOMEM;
25450307a45SLai Jiangshan
25550307a45SLai Jiangshan /* Second: actually write the @print_fmt */
25650307a45SLai Jiangshan __set_enter_print_fmt(entry, print_fmt, len + 1);
25750307a45SLai Jiangshan call->print_fmt = print_fmt;
25850307a45SLai Jiangshan
25950307a45SLai Jiangshan return 0;
26050307a45SLai Jiangshan }
26150307a45SLai Jiangshan
free_syscall_print_fmt(struct trace_event_call * call)2622425bcb9SSteven Rostedt (Red Hat) static void __init free_syscall_print_fmt(struct trace_event_call *call)
26350307a45SLai Jiangshan {
26450307a45SLai Jiangshan struct syscall_metadata *entry = call->data;
26550307a45SLai Jiangshan
26650307a45SLai Jiangshan if (entry->enter_event == call)
26750307a45SLai Jiangshan kfree(call->print_fmt);
26850307a45SLai Jiangshan }
26950307a45SLai Jiangshan
syscall_enter_define_fields(struct trace_event_call * call)2702425bcb9SSteven Rostedt (Red Hat) static int __init syscall_enter_define_fields(struct trace_event_call *call)
271540b7b8dSLi Zefan {
272540b7b8dSLi Zefan struct syscall_trace_enter trace;
27331c16b13SLai Jiangshan struct syscall_metadata *meta = call->data;
274540b7b8dSLi Zefan int offset = offsetof(typeof(trace), args);
27531537cf8SSteven Rostedt (VMware) int ret = 0;
27631537cf8SSteven Rostedt (VMware) int i;
2770f1ef51dSLai Jiangshan
278540b7b8dSLi Zefan for (i = 0; i < meta->nb_args; i++) {
279aeaeae11SFrederic Weisbecker ret = trace_define_field(call, meta->types[i],
280aeaeae11SFrederic Weisbecker meta->args[i], offset,
28143b51eadSLi Zefan sizeof(unsigned long), 0,
28243b51eadSLi Zefan FILTER_OTHER);
28304ae87a5SPeter Zijlstra if (ret)
28404ae87a5SPeter Zijlstra break;
285540b7b8dSLi Zefan offset += sizeof(unsigned long);
286540b7b8dSLi Zefan }
287540b7b8dSLi Zefan
288540b7b8dSLi Zefan return ret;
289540b7b8dSLi Zefan }
290540b7b8dSLi Zefan
ftrace_syscall_enter(void * data,struct pt_regs * regs,long id)29112ab74eeSSteven Rostedt static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id)
292ee08c6ecSFrederic Weisbecker {
29312ab74eeSSteven Rostedt struct trace_array *tr = data;
2947f1d2f82SSteven Rostedt (Red Hat) struct trace_event_file *trace_file;
295bed1ffcaSFrederic Weisbecker struct syscall_trace_enter *entry;
296bed1ffcaSFrederic Weisbecker struct syscall_metadata *sys_data;
297cb1c45fbSJeff Xie struct trace_event_buffer fbuffer;
298d08e4113SSteven Rostedt (Red Hat) unsigned long args[6];
299ee08c6ecSFrederic Weisbecker int syscall_nr;
300f431b634SSteven Rostedt int size;
301ee08c6ecSFrederic Weisbecker
302f431b634SSteven Rostedt syscall_nr = trace_get_syscall_nr(current, regs);
303086ba77aSRabin Vincent if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
304cd0980fcSHendrik Brueckner return;
305d562aff9STom Zanussi
306d562aff9STom Zanussi /* Here we're inside tp handler's rcu_read_lock_sched (__DO_TRACE) */
3077f1d2f82SSteven Rostedt (Red Hat) trace_file = rcu_dereference_sched(tr->enter_syscall_files[syscall_nr]);
3087f1d2f82SSteven Rostedt (Red Hat) if (!trace_file)
309d562aff9STom Zanussi return;
310d562aff9STom Zanussi
31109a5059aSSteven Rostedt (Red Hat) if (trace_trigger_soft_disabled(trace_file))
312fb34a08cSJason Baron return;
313ee08c6ecSFrederic Weisbecker
314bed1ffcaSFrederic Weisbecker sys_data = syscall_nr_to_meta(syscall_nr);
315bed1ffcaSFrederic Weisbecker if (!sys_data)
316bed1ffcaSFrederic Weisbecker return;
317bed1ffcaSFrederic Weisbecker
318bed1ffcaSFrederic Weisbecker size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;
319bed1ffcaSFrederic Weisbecker
320cb1c45fbSJeff Xie entry = trace_event_buffer_reserve(&fbuffer, trace_file, size);
321cb1c45fbSJeff Xie if (!entry)
322bed1ffcaSFrederic Weisbecker return;
323bed1ffcaSFrederic Weisbecker
324cb1c45fbSJeff Xie entry = ring_buffer_event_data(fbuffer.event);
325bed1ffcaSFrederic Weisbecker entry->nr = syscall_nr;
326b35f549dSSteven Rostedt (Red Hat) syscall_get_arguments(current, regs, args);
327d08e4113SSteven Rostedt (Red Hat) memcpy(entry->args, args, sizeof(unsigned long) * sys_data->nb_args);
328bed1ffcaSFrederic Weisbecker
329cb1c45fbSJeff Xie trace_event_buffer_commit(&fbuffer);
330ee08c6ecSFrederic Weisbecker }
331ee08c6ecSFrederic Weisbecker
ftrace_syscall_exit(void * data,struct pt_regs * regs,long ret)33212ab74eeSSteven Rostedt static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret)
333ee08c6ecSFrederic Weisbecker {
33412ab74eeSSteven Rostedt struct trace_array *tr = data;
3357f1d2f82SSteven Rostedt (Red Hat) struct trace_event_file *trace_file;
336bed1ffcaSFrederic Weisbecker struct syscall_trace_exit *entry;
337bed1ffcaSFrederic Weisbecker struct syscall_metadata *sys_data;
338cb1c45fbSJeff Xie struct trace_event_buffer fbuffer;
339ee08c6ecSFrederic Weisbecker int syscall_nr;
340ee08c6ecSFrederic Weisbecker
341f431b634SSteven Rostedt syscall_nr = trace_get_syscall_nr(current, regs);
342086ba77aSRabin Vincent if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
343cd0980fcSHendrik Brueckner return;
344d562aff9STom Zanussi
345d562aff9STom Zanussi /* Here we're inside tp handler's rcu_read_lock_sched (__DO_TRACE()) */
3467f1d2f82SSteven Rostedt (Red Hat) trace_file = rcu_dereference_sched(tr->exit_syscall_files[syscall_nr]);
3477f1d2f82SSteven Rostedt (Red Hat) if (!trace_file)
348d562aff9STom Zanussi return;
349d562aff9STom Zanussi
35009a5059aSSteven Rostedt (Red Hat) if (trace_trigger_soft_disabled(trace_file))
351fb34a08cSJason Baron return;
352ee08c6ecSFrederic Weisbecker
353bed1ffcaSFrederic Weisbecker sys_data = syscall_nr_to_meta(syscall_nr);
354bed1ffcaSFrederic Weisbecker if (!sys_data)
355bed1ffcaSFrederic Weisbecker return;
356bed1ffcaSFrederic Weisbecker
357cb1c45fbSJeff Xie entry = trace_event_buffer_reserve(&fbuffer, trace_file, sizeof(*entry));
358cb1c45fbSJeff Xie if (!entry)
359bed1ffcaSFrederic Weisbecker return;
360bed1ffcaSFrederic Weisbecker
361cb1c45fbSJeff Xie entry = ring_buffer_event_data(fbuffer.event);
362bed1ffcaSFrederic Weisbecker entry->nr = syscall_nr;
363bed1ffcaSFrederic Weisbecker entry->ret = syscall_get_return_value(current, regs);
364bed1ffcaSFrederic Weisbecker
365cb1c45fbSJeff Xie trace_event_buffer_commit(&fbuffer);
366ee08c6ecSFrederic Weisbecker }
367ee08c6ecSFrederic Weisbecker
reg_event_syscall_enter(struct trace_event_file * file,struct trace_event_call * call)3687f1d2f82SSteven Rostedt (Red Hat) static int reg_event_syscall_enter(struct trace_event_file *file,
3692425bcb9SSteven Rostedt (Red Hat) struct trace_event_call *call)
370ee08c6ecSFrederic Weisbecker {
37112ab74eeSSteven Rostedt struct trace_array *tr = file->tr;
372fb34a08cSJason Baron int ret = 0;
373fb34a08cSJason Baron int num;
374ee08c6ecSFrederic Weisbecker
375c252f657SLai Jiangshan num = ((struct syscall_metadata *)call->data)->syscall_nr;
3763773b389SIan Munsie if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
377fb34a08cSJason Baron return -ENOSYS;
378fb34a08cSJason Baron mutex_lock(&syscall_trace_lock);
37912ab74eeSSteven Rostedt if (!tr->sys_refcount_enter)
38012ab74eeSSteven Rostedt ret = register_trace_sys_enter(ftrace_syscall_enter, tr);
3813b8e4273SLi Zefan if (!ret) {
382d562aff9STom Zanussi rcu_assign_pointer(tr->enter_syscall_files[num], file);
38312ab74eeSSteven Rostedt tr->sys_refcount_enter++;
384fb34a08cSJason Baron }
385fb34a08cSJason Baron mutex_unlock(&syscall_trace_lock);
386fb34a08cSJason Baron return ret;
387ee08c6ecSFrederic Weisbecker }
388ee08c6ecSFrederic Weisbecker
unreg_event_syscall_enter(struct trace_event_file * file,struct trace_event_call * call)3897f1d2f82SSteven Rostedt (Red Hat) static void unreg_event_syscall_enter(struct trace_event_file *file,
3902425bcb9SSteven Rostedt (Red Hat) struct trace_event_call *call)
391ee08c6ecSFrederic Weisbecker {
39212ab74eeSSteven Rostedt struct trace_array *tr = file->tr;
393fb34a08cSJason Baron int num;
394fb34a08cSJason Baron
395c252f657SLai Jiangshan num = ((struct syscall_metadata *)call->data)->syscall_nr;
3963773b389SIan Munsie if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
397fb34a08cSJason Baron return;
398fb34a08cSJason Baron mutex_lock(&syscall_trace_lock);
39912ab74eeSSteven Rostedt tr->sys_refcount_enter--;
400fb5a613bSAndreea-Cristina Bernat RCU_INIT_POINTER(tr->enter_syscall_files[num], NULL);
40112ab74eeSSteven Rostedt if (!tr->sys_refcount_enter)
40212ab74eeSSteven Rostedt unregister_trace_sys_enter(ftrace_syscall_enter, tr);
403fb34a08cSJason Baron mutex_unlock(&syscall_trace_lock);
404ee08c6ecSFrederic Weisbecker }
405ee08c6ecSFrederic Weisbecker
reg_event_syscall_exit(struct trace_event_file * file,struct trace_event_call * call)4067f1d2f82SSteven Rostedt (Red Hat) static int reg_event_syscall_exit(struct trace_event_file *file,
4072425bcb9SSteven Rostedt (Red Hat) struct trace_event_call *call)
408fb34a08cSJason Baron {
40912ab74eeSSteven Rostedt struct trace_array *tr = file->tr;
410fb34a08cSJason Baron int ret = 0;
411fb34a08cSJason Baron int num;
412fb34a08cSJason Baron
413c252f657SLai Jiangshan num = ((struct syscall_metadata *)call->data)->syscall_nr;
4143773b389SIan Munsie if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
415fb34a08cSJason Baron return -ENOSYS;
416fb34a08cSJason Baron mutex_lock(&syscall_trace_lock);
41712ab74eeSSteven Rostedt if (!tr->sys_refcount_exit)
41812ab74eeSSteven Rostedt ret = register_trace_sys_exit(ftrace_syscall_exit, tr);
4193b8e4273SLi Zefan if (!ret) {
420d562aff9STom Zanussi rcu_assign_pointer(tr->exit_syscall_files[num], file);
42112ab74eeSSteven Rostedt tr->sys_refcount_exit++;
422fb34a08cSJason Baron }
423fb34a08cSJason Baron mutex_unlock(&syscall_trace_lock);
424fb34a08cSJason Baron return ret;
425fb34a08cSJason Baron }
426fb34a08cSJason Baron
unreg_event_syscall_exit(struct trace_event_file * file,struct trace_event_call * call)4277f1d2f82SSteven Rostedt (Red Hat) static void unreg_event_syscall_exit(struct trace_event_file *file,
4282425bcb9SSteven Rostedt (Red Hat) struct trace_event_call *call)
429fb34a08cSJason Baron {
43012ab74eeSSteven Rostedt struct trace_array *tr = file->tr;
431fb34a08cSJason Baron int num;
432fb34a08cSJason Baron
433c252f657SLai Jiangshan num = ((struct syscall_metadata *)call->data)->syscall_nr;
4343773b389SIan Munsie if (WARN_ON_ONCE(num < 0 || num >= NR_syscalls))
435fb34a08cSJason Baron return;
436fb34a08cSJason Baron mutex_lock(&syscall_trace_lock);
43712ab74eeSSteven Rostedt tr->sys_refcount_exit--;
438fb5a613bSAndreea-Cristina Bernat RCU_INIT_POINTER(tr->exit_syscall_files[num], NULL);
43912ab74eeSSteven Rostedt if (!tr->sys_refcount_exit)
44012ab74eeSSteven Rostedt unregister_trace_sys_exit(ftrace_syscall_exit, tr);
441fb34a08cSJason Baron mutex_unlock(&syscall_trace_lock);
442fb34a08cSJason Baron }
443fb34a08cSJason Baron
init_syscall_trace(struct trace_event_call * call)4442425bcb9SSteven Rostedt (Red Hat) static int __init init_syscall_trace(struct trace_event_call *call)
445a1301da0SLai Jiangshan {
446a1301da0SLai Jiangshan int id;
447ba976970SIan Munsie int num;
448ba976970SIan Munsie
449ba976970SIan Munsie num = ((struct syscall_metadata *)call->data)->syscall_nr;
450ba976970SIan Munsie if (num < 0 || num >= NR_syscalls) {
451ba976970SIan Munsie pr_debug("syscall %s metadata not mapped, disabling ftrace event\n",
452ba976970SIan Munsie ((struct syscall_metadata *)call->data)->name);
453ba976970SIan Munsie return -ENOSYS;
454ba976970SIan Munsie }
455a1301da0SLai Jiangshan
45650307a45SLai Jiangshan if (set_syscall_print_fmt(call) < 0)
45750307a45SLai Jiangshan return -ENOMEM;
45850307a45SLai Jiangshan
459c7ef3a90SSteven Rostedt id = trace_event_raw_init(call);
460c7ef3a90SSteven Rostedt
461c7ef3a90SSteven Rostedt if (id < 0) {
46250307a45SLai Jiangshan free_syscall_print_fmt(call);
463c7ef3a90SSteven Rostedt return id;
46450307a45SLai Jiangshan }
465c7ef3a90SSteven Rostedt
466c7ef3a90SSteven Rostedt return id;
467a1301da0SLai Jiangshan }
468a1301da0SLai Jiangshan
46904ae87a5SPeter Zijlstra static struct trace_event_fields __refdata syscall_enter_fields_array[] = {
47004ae87a5SPeter Zijlstra SYSCALL_FIELD(int, __syscall_nr),
47104ae87a5SPeter Zijlstra { .type = TRACE_FUNCTION_TYPE,
47204ae87a5SPeter Zijlstra .define_fields = syscall_enter_define_fields },
47304ae87a5SPeter Zijlstra {}
47404ae87a5SPeter Zijlstra };
47504ae87a5SPeter Zijlstra
4766f86ab9fSVaibhav Nagarnaik struct trace_event_functions enter_syscall_print_funcs = {
4776f86ab9fSVaibhav Nagarnaik .trace = print_syscall_enter,
4786f86ab9fSVaibhav Nagarnaik };
4796f86ab9fSVaibhav Nagarnaik
4806f86ab9fSVaibhav Nagarnaik struct trace_event_functions exit_syscall_print_funcs = {
4816f86ab9fSVaibhav Nagarnaik .trace = print_syscall_exit,
4826f86ab9fSVaibhav Nagarnaik };
4836f86ab9fSVaibhav Nagarnaik
4842425bcb9SSteven Rostedt (Red Hat) struct trace_event_class __refdata event_class_syscall_enter = {
4856f86ab9fSVaibhav Nagarnaik .system = "syscalls",
4866f86ab9fSVaibhav Nagarnaik .reg = syscall_enter_register,
48704ae87a5SPeter Zijlstra .fields_array = syscall_enter_fields_array,
4886f86ab9fSVaibhav Nagarnaik .get_fields = syscall_get_enter_fields,
4896f86ab9fSVaibhav Nagarnaik .raw_init = init_syscall_trace,
4906f86ab9fSVaibhav Nagarnaik };
4916f86ab9fSVaibhav Nagarnaik
4922425bcb9SSteven Rostedt (Red Hat) struct trace_event_class __refdata event_class_syscall_exit = {
4936f86ab9fSVaibhav Nagarnaik .system = "syscalls",
4946f86ab9fSVaibhav Nagarnaik .reg = syscall_exit_register,
49504ae87a5SPeter Zijlstra .fields_array = (struct trace_event_fields[]){
49604ae87a5SPeter Zijlstra SYSCALL_FIELD(int, __syscall_nr),
49704ae87a5SPeter Zijlstra SYSCALL_FIELD(long, ret),
49804ae87a5SPeter Zijlstra {}
49904ae87a5SPeter Zijlstra },
5006f86ab9fSVaibhav Nagarnaik .fields = LIST_HEAD_INIT(event_class_syscall_exit.fields),
5016f86ab9fSVaibhav Nagarnaik .raw_init = init_syscall_trace,
5026f86ab9fSVaibhav Nagarnaik };
5036f86ab9fSVaibhav Nagarnaik
arch_syscall_addr(int nr)504c763ba06SIan Munsie unsigned long __init __weak arch_syscall_addr(int nr)
505e7b8e675SMike Frysinger {
506e7b8e675SMike Frysinger return (unsigned long)sys_call_table[nr];
507e7b8e675SMike Frysinger }
508e7b8e675SMike Frysinger
init_ftrace_syscalls(void)5095f893b26SSteven Rostedt (Red Hat) void __init init_ftrace_syscalls(void)
510c44fc770SFrederic Weisbecker {
511c44fc770SFrederic Weisbecker struct syscall_metadata *meta;
512c44fc770SFrederic Weisbecker unsigned long addr;
513c44fc770SFrederic Weisbecker int i;
5140e242208SHassan Naveed void *ret;
515c44fc770SFrederic Weisbecker
5160e242208SHassan Naveed if (!IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR)) {
5170e242208SHassan Naveed syscalls_metadata = kcalloc(NR_syscalls,
5180e242208SHassan Naveed sizeof(*syscalls_metadata),
51947b0edcbSThomas Meyer GFP_KERNEL);
520c44fc770SFrederic Weisbecker if (!syscalls_metadata) {
521c44fc770SFrederic Weisbecker WARN_ON(1);
5225f893b26SSteven Rostedt (Red Hat) return;
523c44fc770SFrederic Weisbecker }
5240e242208SHassan Naveed }
525c44fc770SFrederic Weisbecker
526c44fc770SFrederic Weisbecker for (i = 0; i < NR_syscalls; i++) {
527c44fc770SFrederic Weisbecker addr = arch_syscall_addr(i);
528c44fc770SFrederic Weisbecker meta = find_syscall_meta(addr);
529c252f657SLai Jiangshan if (!meta)
530c252f657SLai Jiangshan continue;
531c252f657SLai Jiangshan
532c252f657SLai Jiangshan meta->syscall_nr = i;
5330e242208SHassan Naveed
5340e242208SHassan Naveed if (!IS_ENABLED(CONFIG_HAVE_SPARSE_SYSCALL_NR)) {
535c44fc770SFrederic Weisbecker syscalls_metadata[i] = meta;
5360e242208SHassan Naveed } else {
5370e242208SHassan Naveed ret = xa_store(&syscalls_metadata_sparse, i, meta,
5380e242208SHassan Naveed GFP_KERNEL);
5390e242208SHassan Naveed WARN(xa_is_err(ret),
5400e242208SHassan Naveed "Syscall memory allocation failed\n");
5410e242208SHassan Naveed }
5420e242208SHassan Naveed
543c44fc770SFrederic Weisbecker }
544c44fc770SFrederic Weisbecker }
545c44fc770SFrederic Weisbecker
54607b139c8SLi Zefan #ifdef CONFIG_PERF_EVENTS
54719007a67SFrederic Weisbecker
54897d5a220SFrederic Weisbecker static DECLARE_BITMAP(enabled_perf_enter_syscalls, NR_syscalls);
54997d5a220SFrederic Weisbecker static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
55097d5a220SFrederic Weisbecker static int sys_perf_refcount_enter;
55197d5a220SFrederic Weisbecker static int sys_perf_refcount_exit;
552f4b5ffccSJason Baron
perf_call_bpf_enter(struct trace_event_call * call,struct pt_regs * regs,struct syscall_metadata * sys_data,struct syscall_trace_enter * rec)553e87c6bc3SYonghong Song static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *regs,
554cf5f5ceaSYonghong Song struct syscall_metadata *sys_data,
555e87c6bc3SYonghong Song struct syscall_trace_enter *rec)
556e87c6bc3SYonghong Song {
557cf5f5ceaSYonghong Song struct syscall_tp_t {
558*d3c4db86SYauheni Kaliuta struct trace_entry ent;
559cf5f5ceaSYonghong Song unsigned long syscall_nr;
560609320c8SYonghong Song unsigned long args[SYSCALL_DEFINE_MAXARGS];
561*d3c4db86SYauheni Kaliuta } __aligned(8) param;
562cf5f5ceaSYonghong Song int i;
563cf5f5ceaSYonghong Song
564*d3c4db86SYauheni Kaliuta BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *));
565*d3c4db86SYauheni Kaliuta
566*d3c4db86SYauheni Kaliuta /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. ¶m) */
567cf5f5ceaSYonghong Song *(struct pt_regs **)¶m = regs;
568cf5f5ceaSYonghong Song param.syscall_nr = rec->nr;
569cf5f5ceaSYonghong Song for (i = 0; i < sys_data->nb_args; i++)
570cf5f5ceaSYonghong Song param.args[i] = rec->args[i];
571e87c6bc3SYonghong Song return trace_call_bpf(call, ¶m);
572cf5f5ceaSYonghong Song }
573cf5f5ceaSYonghong Song
perf_syscall_enter(void * ignore,struct pt_regs * regs,long id)57438516ab5SSteven Rostedt static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
575f4b5ffccSJason Baron {
576f4b5ffccSJason Baron struct syscall_metadata *sys_data;
57720ab4425SFrederic Weisbecker struct syscall_trace_enter *rec;
5781c024ecaSPeter Zijlstra struct hlist_head *head;
579d08e4113SSteven Rostedt (Red Hat) unsigned long args[6];
580e87c6bc3SYonghong Song bool valid_prog_array;
581f4b5ffccSJason Baron int syscall_nr;
5824ed7c92dSPeter Zijlstra int rctx;
58319007a67SFrederic Weisbecker int size;
584f4b5ffccSJason Baron
585f431b634SSteven Rostedt syscall_nr = trace_get_syscall_nr(current, regs);
586086ba77aSRabin Vincent if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
58760916a93SWill Deacon return;
58897d5a220SFrederic Weisbecker if (!test_bit(syscall_nr, enabled_perf_enter_syscalls))
589f4b5ffccSJason Baron return;
590f4b5ffccSJason Baron
591f4b5ffccSJason Baron sys_data = syscall_nr_to_meta(syscall_nr);
592f4b5ffccSJason Baron if (!sys_data)
593f4b5ffccSJason Baron return;
594f4b5ffccSJason Baron
595421c7860SOleg Nesterov head = this_cpu_ptr(sys_data->enter_event->perf_events);
596e87c6bc3SYonghong Song valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);
597e87c6bc3SYonghong Song if (!valid_prog_array && hlist_empty(head))
598421c7860SOleg Nesterov return;
599421c7860SOleg Nesterov
60019007a67SFrederic Weisbecker /* get the size after alignment with the u32 buffer size field */
60119007a67SFrederic Weisbecker size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
60219007a67SFrederic Weisbecker size = ALIGN(size + sizeof(u32), sizeof(u64));
60319007a67SFrederic Weisbecker size -= sizeof(u32);
60419007a67SFrederic Weisbecker
6051e1dcd93SAlexei Starovoitov rec = perf_trace_buf_alloc(size, NULL, &rctx);
606430ad5a6SXiao Guangrong if (!rec)
607430ad5a6SXiao Guangrong return;
60820ab4425SFrederic Weisbecker
609e6971969SLi Zefan rec->nr = syscall_nr;
610b35f549dSSteven Rostedt (Red Hat) syscall_get_arguments(current, regs, args);
611d08e4113SSteven Rostedt (Red Hat) memcpy(&rec->args, args, sizeof(unsigned long) * sys_data->nb_args);
612cf5f5ceaSYonghong Song
613e87c6bc3SYonghong Song if ((valid_prog_array &&
614e87c6bc3SYonghong Song !perf_call_bpf_enter(sys_data->enter_event, regs, sys_data, rec)) ||
615cf5f5ceaSYonghong Song hlist_empty(head)) {
616cf5f5ceaSYonghong Song perf_swevent_put_recursion_context(rctx);
617cf5f5ceaSYonghong Song return;
618cf5f5ceaSYonghong Song }
619cf5f5ceaSYonghong Song
6201e1dcd93SAlexei Starovoitov perf_trace_buf_submit(rec, size, rctx,
6211e1dcd93SAlexei Starovoitov sys_data->enter_event->event.type, 1, regs,
6228fd0fbbeSPeter Zijlstra head, NULL);
623f4b5ffccSJason Baron }
624f4b5ffccSJason Baron
perf_sysenter_enable(struct trace_event_call * call)6252425bcb9SSteven Rostedt (Red Hat) static int perf_sysenter_enable(struct trace_event_call *call)
626f4b5ffccSJason Baron {
627f4b5ffccSJason Baron int ret = 0;
628f4b5ffccSJason Baron int num;
629f4b5ffccSJason Baron
6303bbe84e9SLai Jiangshan num = ((struct syscall_metadata *)call->data)->syscall_nr;
631f4b5ffccSJason Baron
632f4b5ffccSJason Baron mutex_lock(&syscall_trace_lock);
63397d5a220SFrederic Weisbecker if (!sys_perf_refcount_enter)
63438516ab5SSteven Rostedt ret = register_trace_sys_enter(perf_syscall_enter, NULL);
635f4b5ffccSJason Baron if (ret) {
636d282b9c0SColin Ian King pr_info("event trace: Could not activate syscall entry trace point");
637f4b5ffccSJason Baron } else {
63897d5a220SFrederic Weisbecker set_bit(num, enabled_perf_enter_syscalls);
63997d5a220SFrederic Weisbecker sys_perf_refcount_enter++;
640f4b5ffccSJason Baron }
641f4b5ffccSJason Baron mutex_unlock(&syscall_trace_lock);
642f4b5ffccSJason Baron return ret;
643f4b5ffccSJason Baron }
644f4b5ffccSJason Baron
perf_sysenter_disable(struct trace_event_call * call)6452425bcb9SSteven Rostedt (Red Hat) static void perf_sysenter_disable(struct trace_event_call *call)
646f4b5ffccSJason Baron {
647f4b5ffccSJason Baron int num;
648f4b5ffccSJason Baron
6493bbe84e9SLai Jiangshan num = ((struct syscall_metadata *)call->data)->syscall_nr;
650f4b5ffccSJason Baron
651f4b5ffccSJason Baron mutex_lock(&syscall_trace_lock);
65297d5a220SFrederic Weisbecker sys_perf_refcount_enter--;
65397d5a220SFrederic Weisbecker clear_bit(num, enabled_perf_enter_syscalls);
65497d5a220SFrederic Weisbecker if (!sys_perf_refcount_enter)
65538516ab5SSteven Rostedt unregister_trace_sys_enter(perf_syscall_enter, NULL);
656f4b5ffccSJason Baron mutex_unlock(&syscall_trace_lock);
657f4b5ffccSJason Baron }
658f4b5ffccSJason Baron
perf_call_bpf_exit(struct trace_event_call * call,struct pt_regs * regs,struct syscall_trace_exit * rec)659e87c6bc3SYonghong Song static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *regs,
660e87c6bc3SYonghong Song struct syscall_trace_exit *rec)
661e87c6bc3SYonghong Song {
662cf5f5ceaSYonghong Song struct syscall_tp_t {
663*d3c4db86SYauheni Kaliuta struct trace_entry ent;
664cf5f5ceaSYonghong Song unsigned long syscall_nr;
665cf5f5ceaSYonghong Song unsigned long ret;
666*d3c4db86SYauheni Kaliuta } __aligned(8) param;
667cf5f5ceaSYonghong Song
668*d3c4db86SYauheni Kaliuta /* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. ¶m) */
669cf5f5ceaSYonghong Song *(struct pt_regs **)¶m = regs;
670cf5f5ceaSYonghong Song param.syscall_nr = rec->nr;
671cf5f5ceaSYonghong Song param.ret = rec->ret;
672e87c6bc3SYonghong Song return trace_call_bpf(call, ¶m);
673cf5f5ceaSYonghong Song }
674cf5f5ceaSYonghong Song
perf_syscall_exit(void * ignore,struct pt_regs * regs,long ret)67538516ab5SSteven Rostedt static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
676f4b5ffccSJason Baron {
677f4b5ffccSJason Baron struct syscall_metadata *sys_data;
67820ab4425SFrederic Weisbecker struct syscall_trace_exit *rec;
6791c024ecaSPeter Zijlstra struct hlist_head *head;
680e87c6bc3SYonghong Song bool valid_prog_array;
681f4b5ffccSJason Baron int syscall_nr;
6824ed7c92dSPeter Zijlstra int rctx;
68320ab4425SFrederic Weisbecker int size;
684f4b5ffccSJason Baron
685f431b634SSteven Rostedt syscall_nr = trace_get_syscall_nr(current, regs);
686086ba77aSRabin Vincent if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
68760916a93SWill Deacon return;
68897d5a220SFrederic Weisbecker if (!test_bit(syscall_nr, enabled_perf_exit_syscalls))
689f4b5ffccSJason Baron return;
690f4b5ffccSJason Baron
691f4b5ffccSJason Baron sys_data = syscall_nr_to_meta(syscall_nr);
692f4b5ffccSJason Baron if (!sys_data)
693f4b5ffccSJason Baron return;
694f4b5ffccSJason Baron
695421c7860SOleg Nesterov head = this_cpu_ptr(sys_data->exit_event->perf_events);
696e87c6bc3SYonghong Song valid_prog_array = bpf_prog_array_valid(sys_data->exit_event);
697e87c6bc3SYonghong Song if (!valid_prog_array && hlist_empty(head))
698421c7860SOleg Nesterov return;
699421c7860SOleg Nesterov
70020ab4425SFrederic Weisbecker /* We can probably do that at build time */
70120ab4425SFrederic Weisbecker size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
70220ab4425SFrederic Weisbecker size -= sizeof(u32);
70319007a67SFrederic Weisbecker
7041e1dcd93SAlexei Starovoitov rec = perf_trace_buf_alloc(size, NULL, &rctx);
705430ad5a6SXiao Guangrong if (!rec)
706430ad5a6SXiao Guangrong return;
707ce71b9dfSFrederic Weisbecker
70820ab4425SFrederic Weisbecker rec->nr = syscall_nr;
70920ab4425SFrederic Weisbecker rec->ret = syscall_get_return_value(current, regs);
710cf5f5ceaSYonghong Song
711e87c6bc3SYonghong Song if ((valid_prog_array &&
712e87c6bc3SYonghong Song !perf_call_bpf_exit(sys_data->exit_event, regs, rec)) ||
713cf5f5ceaSYonghong Song hlist_empty(head)) {
714cf5f5ceaSYonghong Song perf_swevent_put_recursion_context(rctx);
715cf5f5ceaSYonghong Song return;
716cf5f5ceaSYonghong Song }
717cf5f5ceaSYonghong Song
7181e1dcd93SAlexei Starovoitov perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type,
7198fd0fbbeSPeter Zijlstra 1, regs, head, NULL);
720f4b5ffccSJason Baron }
721f4b5ffccSJason Baron
perf_sysexit_enable(struct trace_event_call * call)7222425bcb9SSteven Rostedt (Red Hat) static int perf_sysexit_enable(struct trace_event_call *call)
723f4b5ffccSJason Baron {
724f4b5ffccSJason Baron int ret = 0;
725f4b5ffccSJason Baron int num;
726f4b5ffccSJason Baron
7273bbe84e9SLai Jiangshan num = ((struct syscall_metadata *)call->data)->syscall_nr;
728f4b5ffccSJason Baron
729f4b5ffccSJason Baron mutex_lock(&syscall_trace_lock);
73097d5a220SFrederic Weisbecker if (!sys_perf_refcount_exit)
73138516ab5SSteven Rostedt ret = register_trace_sys_exit(perf_syscall_exit, NULL);
732f4b5ffccSJason Baron if (ret) {
733d282b9c0SColin Ian King pr_info("event trace: Could not activate syscall exit trace point");
734f4b5ffccSJason Baron } else {
73597d5a220SFrederic Weisbecker set_bit(num, enabled_perf_exit_syscalls);
73697d5a220SFrederic Weisbecker sys_perf_refcount_exit++;
737f4b5ffccSJason Baron }
738f4b5ffccSJason Baron mutex_unlock(&syscall_trace_lock);
739f4b5ffccSJason Baron return ret;
740f4b5ffccSJason Baron }
741f4b5ffccSJason Baron
perf_sysexit_disable(struct trace_event_call * call)7422425bcb9SSteven Rostedt (Red Hat) static void perf_sysexit_disable(struct trace_event_call *call)
743f4b5ffccSJason Baron {
744f4b5ffccSJason Baron int num;
745f4b5ffccSJason Baron
7463bbe84e9SLai Jiangshan num = ((struct syscall_metadata *)call->data)->syscall_nr;
747f4b5ffccSJason Baron
748f4b5ffccSJason Baron mutex_lock(&syscall_trace_lock);
74997d5a220SFrederic Weisbecker sys_perf_refcount_exit--;
75097d5a220SFrederic Weisbecker clear_bit(num, enabled_perf_exit_syscalls);
75197d5a220SFrederic Weisbecker if (!sys_perf_refcount_exit)
75238516ab5SSteven Rostedt unregister_trace_sys_exit(perf_syscall_exit, NULL);
753f4b5ffccSJason Baron mutex_unlock(&syscall_trace_lock);
754f4b5ffccSJason Baron }
755f4b5ffccSJason Baron
75607b139c8SLi Zefan #endif /* CONFIG_PERF_EVENTS */
757f4b5ffccSJason Baron
syscall_enter_register(struct trace_event_call * event,enum trace_reg type,void * data)7582425bcb9SSteven Rostedt (Red Hat) static int syscall_enter_register(struct trace_event_call *event,
759ceec0b6fSJiri Olsa enum trace_reg type, void *data)
7602239291aSSteven Rostedt {
7617f1d2f82SSteven Rostedt (Red Hat) struct trace_event_file *file = data;
76212ab74eeSSteven Rostedt
7632239291aSSteven Rostedt switch (type) {
7642239291aSSteven Rostedt case TRACE_REG_REGISTER:
76512ab74eeSSteven Rostedt return reg_event_syscall_enter(file, event);
7662239291aSSteven Rostedt case TRACE_REG_UNREGISTER:
76712ab74eeSSteven Rostedt unreg_event_syscall_enter(file, event);
7682239291aSSteven Rostedt return 0;
7692239291aSSteven Rostedt
7702239291aSSteven Rostedt #ifdef CONFIG_PERF_EVENTS
7712239291aSSteven Rostedt case TRACE_REG_PERF_REGISTER:
7722239291aSSteven Rostedt return perf_sysenter_enable(event);
7732239291aSSteven Rostedt case TRACE_REG_PERF_UNREGISTER:
7742239291aSSteven Rostedt perf_sysenter_disable(event);
7752239291aSSteven Rostedt return 0;
776ceec0b6fSJiri Olsa case TRACE_REG_PERF_OPEN:
777ceec0b6fSJiri Olsa case TRACE_REG_PERF_CLOSE:
778489c75c3SJiri Olsa case TRACE_REG_PERF_ADD:
779489c75c3SJiri Olsa case TRACE_REG_PERF_DEL:
780ceec0b6fSJiri Olsa return 0;
7812239291aSSteven Rostedt #endif
7822239291aSSteven Rostedt }
7832239291aSSteven Rostedt return 0;
7842239291aSSteven Rostedt }
7852239291aSSteven Rostedt
syscall_exit_register(struct trace_event_call * event,enum trace_reg type,void * data)7862425bcb9SSteven Rostedt (Red Hat) static int syscall_exit_register(struct trace_event_call *event,
787ceec0b6fSJiri Olsa enum trace_reg type, void *data)
7882239291aSSteven Rostedt {
7897f1d2f82SSteven Rostedt (Red Hat) struct trace_event_file *file = data;
79012ab74eeSSteven Rostedt
7912239291aSSteven Rostedt switch (type) {
7922239291aSSteven Rostedt case TRACE_REG_REGISTER:
79312ab74eeSSteven Rostedt return reg_event_syscall_exit(file, event);
7942239291aSSteven Rostedt case TRACE_REG_UNREGISTER:
79512ab74eeSSteven Rostedt unreg_event_syscall_exit(file, event);
7962239291aSSteven Rostedt return 0;
7972239291aSSteven Rostedt
7982239291aSSteven Rostedt #ifdef CONFIG_PERF_EVENTS
7992239291aSSteven Rostedt case TRACE_REG_PERF_REGISTER:
8002239291aSSteven Rostedt return perf_sysexit_enable(event);
8012239291aSSteven Rostedt case TRACE_REG_PERF_UNREGISTER:
8022239291aSSteven Rostedt perf_sysexit_disable(event);
8032239291aSSteven Rostedt return 0;
804ceec0b6fSJiri Olsa case TRACE_REG_PERF_OPEN:
805ceec0b6fSJiri Olsa case TRACE_REG_PERF_CLOSE:
806489c75c3SJiri Olsa case TRACE_REG_PERF_ADD:
807489c75c3SJiri Olsa case TRACE_REG_PERF_DEL:
808ceec0b6fSJiri Olsa return 0;
8092239291aSSteven Rostedt #endif
8102239291aSSteven Rostedt }
8112239291aSSteven Rostedt return 0;
8122239291aSSteven Rostedt }
813