1bcea3f96SSteven Rostedt (VMware) // SPDX-License-Identifier: GPL-2.0
2f3f096cfSSrikar Dronamraju /*
3f3f096cfSSrikar Dronamraju * uprobes-based tracing events
4f3f096cfSSrikar Dronamraju *
5f3f096cfSSrikar Dronamraju * Copyright (C) IBM Corporation, 2010-2012
6f3f096cfSSrikar Dronamraju * Author: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
7f3f096cfSSrikar Dronamraju */
8ea6eb5e7SAndreas Ziegler #define pr_fmt(fmt) "trace_uprobe: " fmt
9f3f096cfSSrikar Dronamraju
10aef2fedaSJakub Kicinski #include <linux/bpf-cgroup.h>
1117911ff3SSteven Rostedt (VMware) #include <linux/security.h>
120597c49cSMasami Hiramatsu #include <linux/ctype.h>
13f3f096cfSSrikar Dronamraju #include <linux/module.h>
14f3f096cfSSrikar Dronamraju #include <linux/uaccess.h>
15f3f096cfSSrikar Dronamraju #include <linux/uprobes.h>
16f3f096cfSSrikar Dronamraju #include <linux/namei.h>
17b2e902f0SAndy Shevchenko #include <linux/string.h>
18b2d09103SIngo Molnar #include <linux/rculist.h>
198c7dcb84SDelyan Kratunov #include <linux/filter.h>
20f3f096cfSSrikar Dronamraju
210597c49cSMasami Hiramatsu #include "trace_dynevent.h"
22f3f096cfSSrikar Dronamraju #include "trace_probe.h"
2353305928SMasami Hiramatsu #include "trace_probe_tmpl.h"
24f3f096cfSSrikar Dronamraju
25f3f096cfSSrikar Dronamraju #define UPROBE_EVENT_SYSTEM "uprobes"
26f3f096cfSSrikar Dronamraju
27457d1772SOleg Nesterov struct uprobe_trace_entry_head {
28457d1772SOleg Nesterov struct trace_entry ent;
29457d1772SOleg Nesterov unsigned long vaddr[];
30457d1772SOleg Nesterov };
31457d1772SOleg Nesterov
32457d1772SOleg Nesterov #define SIZEOF_TRACE_ENTRY(is_return) \
33457d1772SOleg Nesterov (sizeof(struct uprobe_trace_entry_head) + \
34457d1772SOleg Nesterov sizeof(unsigned long) * (is_return ? 2 : 1))
35457d1772SOleg Nesterov
36457d1772SOleg Nesterov #define DATAOF_TRACE_ENTRY(entry, is_return) \
37457d1772SOleg Nesterov ((void*)(entry) + SIZEOF_TRACE_ENTRY(is_return))
38457d1772SOleg Nesterov
39d262271dSMasami Hiramatsu static int trace_uprobe_create(const char *raw_command);
400597c49cSMasami Hiramatsu static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev);
410597c49cSMasami Hiramatsu static int trace_uprobe_release(struct dyn_event *ev);
420597c49cSMasami Hiramatsu static bool trace_uprobe_is_busy(struct dyn_event *ev);
430597c49cSMasami Hiramatsu static bool trace_uprobe_match(const char *system, const char *event,
4430199137SMasami Hiramatsu int argc, const char **argv, struct dyn_event *ev);
450597c49cSMasami Hiramatsu
460597c49cSMasami Hiramatsu static struct dyn_event_operations trace_uprobe_ops = {
470597c49cSMasami Hiramatsu .create = trace_uprobe_create,
480597c49cSMasami Hiramatsu .show = trace_uprobe_show,
490597c49cSMasami Hiramatsu .is_busy = trace_uprobe_is_busy,
500597c49cSMasami Hiramatsu .free = trace_uprobe_release,
510597c49cSMasami Hiramatsu .match = trace_uprobe_match,
520597c49cSMasami Hiramatsu };
530597c49cSMasami Hiramatsu
54f3f096cfSSrikar Dronamraju /*
55f3f096cfSSrikar Dronamraju * uprobe event core functions
56f3f096cfSSrikar Dronamraju */
57f3f096cfSSrikar Dronamraju struct trace_uprobe {
580597c49cSMasami Hiramatsu struct dyn_event devent;
59a932b738SOleg Nesterov struct uprobe_consumer consumer;
600c92c7a3SSong Liu struct path path;
61f3f096cfSSrikar Dronamraju struct inode *inode;
62f3f096cfSSrikar Dronamraju char *filename;
63f3f096cfSSrikar Dronamraju unsigned long offset;
641cc33161SRavi Bangoria unsigned long ref_ctr_offset;
65f3f096cfSSrikar Dronamraju unsigned long nhit;
6614577c39SNamhyung Kim struct trace_probe tp;
67f3f096cfSSrikar Dronamraju };
68f3f096cfSSrikar Dronamraju
is_trace_uprobe(struct dyn_event * ev)690597c49cSMasami Hiramatsu static bool is_trace_uprobe(struct dyn_event *ev)
700597c49cSMasami Hiramatsu {
710597c49cSMasami Hiramatsu return ev->ops == &trace_uprobe_ops;
720597c49cSMasami Hiramatsu }
730597c49cSMasami Hiramatsu
to_trace_uprobe(struct dyn_event * ev)740597c49cSMasami Hiramatsu static struct trace_uprobe *to_trace_uprobe(struct dyn_event *ev)
750597c49cSMasami Hiramatsu {
760597c49cSMasami Hiramatsu return container_of(ev, struct trace_uprobe, devent);
770597c49cSMasami Hiramatsu }
780597c49cSMasami Hiramatsu
790597c49cSMasami Hiramatsu /**
800597c49cSMasami Hiramatsu * for_each_trace_uprobe - iterate over the trace_uprobe list
810597c49cSMasami Hiramatsu * @pos: the struct trace_uprobe * for each entry
820597c49cSMasami Hiramatsu * @dpos: the struct dyn_event * to use as a loop cursor
830597c49cSMasami Hiramatsu */
840597c49cSMasami Hiramatsu #define for_each_trace_uprobe(pos, dpos) \
850597c49cSMasami Hiramatsu for_each_dyn_event(dpos) \
860597c49cSMasami Hiramatsu if (is_trace_uprobe(dpos) && (pos = to_trace_uprobe(dpos)))
870597c49cSMasami Hiramatsu
88f3f096cfSSrikar Dronamraju static int register_uprobe_event(struct trace_uprobe *tu);
89c6c2401dSSteven Rostedt (Red Hat) static int unregister_uprobe_event(struct trace_uprobe *tu);
90f3f096cfSSrikar Dronamraju
91b7e0bf34SNamhyung Kim static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs);
92b7e0bf34SNamhyung Kim static int uretprobe_dispatcher(struct uprobe_consumer *con,
93b7e0bf34SNamhyung Kim unsigned long func, struct pt_regs *regs);
94b7e0bf34SNamhyung Kim
95b7e0bf34SNamhyung Kim #ifdef CONFIG_STACK_GROWSUP
adjust_stack_addr(unsigned long addr,unsigned int n)96f3f096cfSSrikar Dronamraju static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
97c1ae5c75SOleg Nesterov {
98c1ae5c75SOleg Nesterov return addr - (n * sizeof(long));
99f3f096cfSSrikar Dronamraju }
1003fd996a2SNamhyung Kim #else
adjust_stack_addr(unsigned long addr,unsigned int n)1013fd996a2SNamhyung Kim static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
1023fd996a2SNamhyung Kim {
1033fd996a2SNamhyung Kim return addr + (n * sizeof(long));
1043fd996a2SNamhyung Kim }
1053fd996a2SNamhyung Kim #endif
1063fd996a2SNamhyung Kim
get_user_stack_nth(struct pt_regs * regs,unsigned int n)1073fd996a2SNamhyung Kim static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n)
1083fd996a2SNamhyung Kim {
1093fd996a2SNamhyung Kim unsigned long ret;
1103fd996a2SNamhyung Kim unsigned long addr = user_stack_pointer(regs);
1113fd996a2SNamhyung Kim
1123fd996a2SNamhyung Kim addr = adjust_stack_addr(addr, n);
1133fd996a2SNamhyung Kim
1143fd996a2SNamhyung Kim if (copy_from_user(&ret, (void __force __user *) addr, sizeof(ret)))
1153fd996a2SNamhyung Kim return 0;
1163fd996a2SNamhyung Kim
1173fd996a2SNamhyung Kim return ret;
1183fd996a2SNamhyung Kim }
1193fd996a2SNamhyung Kim
1203fd996a2SNamhyung Kim /*
1213fd996a2SNamhyung Kim * Uprobes-specific fetch functions
1223fd996a2SNamhyung Kim */
1233fd996a2SNamhyung Kim static nokprobe_inline int
probe_mem_read(void * dest,void * src,size_t size)1243fd996a2SNamhyung Kim probe_mem_read(void *dest, void *src, size_t size)
1253fd996a2SNamhyung Kim {
1263fd996a2SNamhyung Kim void __user *vaddr = (void __force __user *)src;
1273fd996a2SNamhyung Kim
12853305928SMasami Hiramatsu return copy_from_user(dest, vaddr, size) ? -EFAULT : 0;
1299b960a38SMasami Hiramatsu }
13053305928SMasami Hiramatsu
13153305928SMasami Hiramatsu static nokprobe_inline int
probe_mem_read_user(void * dest,void * src,size_t size)1323fd996a2SNamhyung Kim probe_mem_read_user(void *dest, void *src, size_t size)
133f3f58935SMasami Hiramatsu {
1345baaa59eSNamhyung Kim return probe_mem_read(dest, src, size);
135e65f7ae7SMasami Hiramatsu }
136e65f7ae7SMasami Hiramatsu
137e65f7ae7SMasami Hiramatsu /*
138e65f7ae7SMasami Hiramatsu * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
139e65f7ae7SMasami Hiramatsu * length and relative data location.
140e65f7ae7SMasami Hiramatsu */
141e65f7ae7SMasami Hiramatsu static nokprobe_inline int
fetch_store_string(unsigned long addr,void * dest,void * base)1425baaa59eSNamhyung Kim fetch_store_string(unsigned long addr, void *dest, void *base)
1435baaa59eSNamhyung Kim {
1445baaa59eSNamhyung Kim long ret;
1455baaa59eSNamhyung Kim u32 loc = *(u32 *)dest;
1469178412dSMasami Hiramatsu int maxlen = get_loc_len(loc);
1479178412dSMasami Hiramatsu u8 *dst = get_loc_data(dest, base);
1485baaa59eSNamhyung Kim void __user *src = (void __force __user *) addr;
1495baaa59eSNamhyung Kim
1509178412dSMasami Hiramatsu if (unlikely(!maxlen))
1519178412dSMasami Hiramatsu return -ENOMEM;
1529178412dSMasami Hiramatsu
1535baaa59eSNamhyung Kim if (addr == FETCH_TOKEN_COMM)
1545baaa59eSNamhyung Kim ret = strlcpy(dst, current->comm, maxlen);
1559178412dSMasami Hiramatsu else
1569178412dSMasami Hiramatsu ret = strncpy_from_user(dst, src, maxlen);
1575baaa59eSNamhyung Kim if (ret >= 0) {
1584dd537acSMasami Hiramatsu if (ret == maxlen)
1594dd537acSMasami Hiramatsu dst[ret - 1] = '\0';
1604dd537acSMasami Hiramatsu else
1615baaa59eSNamhyung Kim /*
1629178412dSMasami Hiramatsu * Include the terminating null byte. In this case it
16350268a3dSMasami Hiramatsu * was copied by strncpy_from_user but not accounted
1649178412dSMasami Hiramatsu * for in ret.
1650722069aSAndreas Ziegler */
1660722069aSAndreas Ziegler ret++;
1670722069aSAndreas Ziegler *(u32 *)dest = make_data_loc(ret, (void *)dst - base);
1680722069aSAndreas Ziegler } else
1690722069aSAndreas Ziegler *(u32 *)dest = make_data_loc(0, (void *)dst - base);
1700722069aSAndreas Ziegler
1710722069aSAndreas Ziegler return ret;
1729178412dSMasami Hiramatsu }
173797311bcSMasami Hiramatsu (Google)
174797311bcSMasami Hiramatsu (Google) static nokprobe_inline int
fetch_store_string_user(unsigned long addr,void * dest,void * base)1759178412dSMasami Hiramatsu fetch_store_string_user(unsigned long addr, void *dest, void *base)
1769178412dSMasami Hiramatsu {
1775baaa59eSNamhyung Kim return fetch_store_string(addr, dest, base);
1785baaa59eSNamhyung Kim }
17988903c46SMasami Hiramatsu
18088903c46SMasami Hiramatsu /* Return the length of string -- including null terminal byte */
18188903c46SMasami Hiramatsu static nokprobe_inline int
fetch_store_strlen(unsigned long addr)18288903c46SMasami Hiramatsu fetch_store_strlen(unsigned long addr)
18388903c46SMasami Hiramatsu {
18488903c46SMasami Hiramatsu int len;
18553305928SMasami Hiramatsu void __user *vaddr = (void __force __user *) addr;
1869178412dSMasami Hiramatsu
1879178412dSMasami Hiramatsu if (addr == FETCH_TOKEN_COMM)
1885baaa59eSNamhyung Kim len = strlen(current->comm) + 1;
1895baaa59eSNamhyung Kim else
1905baaa59eSNamhyung Kim len = strnlen_user(vaddr, MAX_STRING_SIZE);
1915baaa59eSNamhyung Kim
1924dd537acSMasami Hiramatsu return (len > MAX_STRING_SIZE) ? 0 : len;
1934dd537acSMasami Hiramatsu }
1944dd537acSMasami Hiramatsu
1955baaa59eSNamhyung Kim static nokprobe_inline int
fetch_store_strlen_user(unsigned long addr)1965baaa59eSNamhyung Kim fetch_store_strlen_user(unsigned long addr)
1979178412dSMasami Hiramatsu {
1985baaa59eSNamhyung Kim return fetch_store_strlen(addr);
1993fd996a2SNamhyung Kim }
20088903c46SMasami Hiramatsu
translate_user_vaddr(unsigned long file_offset)20188903c46SMasami Hiramatsu static unsigned long translate_user_vaddr(unsigned long file_offset)
20288903c46SMasami Hiramatsu {
20388903c46SMasami Hiramatsu unsigned long base_addr;
20488903c46SMasami Hiramatsu struct uprobe_dispatch_data *udd;
20588903c46SMasami Hiramatsu
20653305928SMasami Hiramatsu udd = (void *) current->utask->vaddr;
207b7e0bf34SNamhyung Kim
208b7e0bf34SNamhyung Kim base_addr = udd->bp_addr - udd->tu->offset;
209b7e0bf34SNamhyung Kim return base_addr + file_offset;
210b7e0bf34SNamhyung Kim }
211b7e0bf34SNamhyung Kim
212b7e0bf34SNamhyung Kim /* Note that we don't verify it, since the code does not come from user space */
213b7e0bf34SNamhyung Kim static int
process_fetch_insn(struct fetch_insn * code,void * rec,void * dest,void * base)21453305928SMasami Hiramatsu process_fetch_insn(struct fetch_insn *code, void *rec, void *dest,
215b7e0bf34SNamhyung Kim void *base)
216b7e0bf34SNamhyung Kim {
21753305928SMasami Hiramatsu struct pt_regs *regs = rec;
21853305928SMasami Hiramatsu unsigned long val;
2198565a45dSSteven Rostedt (VMware) int ret;
2209178412dSMasami Hiramatsu
22153305928SMasami Hiramatsu /* 1st stage: get value from context */
2228565a45dSSteven Rostedt (VMware) switch (code->op) {
22353305928SMasami Hiramatsu case FETCH_OP_REG:
224bd78acc8SSong Chen val = regs_get_register(regs, code->param);
22553305928SMasami Hiramatsu break;
22653305928SMasami Hiramatsu case FETCH_OP_STACK:
22753305928SMasami Hiramatsu val = get_user_stack_nth(regs, code->param);
22853305928SMasami Hiramatsu break;
22953305928SMasami Hiramatsu case FETCH_OP_STACKP:
23053305928SMasami Hiramatsu val = user_stack_pointer(regs);
23153305928SMasami Hiramatsu break;
23253305928SMasami Hiramatsu case FETCH_OP_RETVAL:
23353305928SMasami Hiramatsu val = regs_return_value(regs);
23453305928SMasami Hiramatsu break;
23553305928SMasami Hiramatsu case FETCH_OP_COMM:
23653305928SMasami Hiramatsu val = FETCH_TOKEN_COMM;
23753305928SMasami Hiramatsu break;
23853305928SMasami Hiramatsu case FETCH_OP_FOFFS:
23953305928SMasami Hiramatsu val = translate_user_vaddr(code->immediate);
2404dd537acSMasami Hiramatsu break;
2414dd537acSMasami Hiramatsu default:
2424dd537acSMasami Hiramatsu ret = process_common_fetch_insn(code, &val);
24353305928SMasami Hiramatsu if (ret < 0)
24453305928SMasami Hiramatsu return ret;
24553305928SMasami Hiramatsu }
24653305928SMasami Hiramatsu code++;
247bd78acc8SSong Chen
248bd78acc8SSong Chen return process_fetch_insn_bottom(code, val, dest, base);
249bd78acc8SSong Chen }
NOKPROBE_SYMBOL(process_fetch_insn)25053305928SMasami Hiramatsu NOKPROBE_SYMBOL(process_fetch_insn)
25153305928SMasami Hiramatsu
25253305928SMasami Hiramatsu static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter)
2539b960a38SMasami Hiramatsu {
25453305928SMasami Hiramatsu rwlock_init(&filter->rwlock);
25553305928SMasami Hiramatsu filter->nr_systemwide = 0;
25653305928SMasami Hiramatsu INIT_LIST_HEAD(&filter->perf_events);
257736288baSOleg Nesterov }
258736288baSOleg Nesterov
uprobe_filter_is_empty(struct trace_uprobe_filter * filter)259736288baSOleg Nesterov static inline bool uprobe_filter_is_empty(struct trace_uprobe_filter *filter)
260736288baSOleg Nesterov {
261736288baSOleg Nesterov return !filter->nr_systemwide && list_empty(&filter->perf_events);
262736288baSOleg Nesterov }
263736288baSOleg Nesterov
is_ret_probe(struct trace_uprobe * tu)264736288baSOleg Nesterov static inline bool is_ret_probe(struct trace_uprobe *tu)
265736288baSOleg Nesterov {
266736288baSOleg Nesterov return tu->consumer.ret_handler != NULL;
267736288baSOleg Nesterov }
268736288baSOleg Nesterov
trace_uprobe_is_busy(struct dyn_event * ev)269c1ae5c75SOleg Nesterov static bool trace_uprobe_is_busy(struct dyn_event *ev)
270c1ae5c75SOleg Nesterov {
271c1ae5c75SOleg Nesterov struct trace_uprobe *tu = to_trace_uprobe(ev);
272c1ae5c75SOleg Nesterov
273c1ae5c75SOleg Nesterov return trace_probe_is_enabled(&tu->tp);
2740597c49cSMasami Hiramatsu }
2750597c49cSMasami Hiramatsu
trace_uprobe_match_command_head(struct trace_uprobe * tu,int argc,const char ** argv)2760597c49cSMasami Hiramatsu static bool trace_uprobe_match_command_head(struct trace_uprobe *tu,
2770597c49cSMasami Hiramatsu int argc, const char **argv)
2780597c49cSMasami Hiramatsu {
2790597c49cSMasami Hiramatsu char buf[MAX_ARGSTR_LEN + 1];
2800597c49cSMasami Hiramatsu int len;
281ab10d69eSMasami Hiramatsu
282ab10d69eSMasami Hiramatsu if (!argc)
283ab10d69eSMasami Hiramatsu return true;
284ab10d69eSMasami Hiramatsu
285ab10d69eSMasami Hiramatsu len = strlen(tu->filename);
286ab10d69eSMasami Hiramatsu if (strncmp(tu->filename, argv[0], len) || argv[0][len] != ':')
287ab10d69eSMasami Hiramatsu return false;
288ab10d69eSMasami Hiramatsu
289ab10d69eSMasami Hiramatsu if (tu->ref_ctr_offset == 0)
290ab10d69eSMasami Hiramatsu snprintf(buf, sizeof(buf), "0x%0*lx",
291ab10d69eSMasami Hiramatsu (int)(sizeof(void *) * 2), tu->offset);
292ab10d69eSMasami Hiramatsu else
293ab10d69eSMasami Hiramatsu snprintf(buf, sizeof(buf), "0x%0*lx(0x%lx)",
294ab10d69eSMasami Hiramatsu (int)(sizeof(void *) * 2), tu->offset,
295ab10d69eSMasami Hiramatsu tu->ref_ctr_offset);
296ab10d69eSMasami Hiramatsu if (strcmp(buf, &argv[0][len + 1]))
297ab10d69eSMasami Hiramatsu return false;
298ab10d69eSMasami Hiramatsu
299ab10d69eSMasami Hiramatsu argc--; argv++;
300ab10d69eSMasami Hiramatsu
301ab10d69eSMasami Hiramatsu return trace_probe_match_command_args(&tu->tp, argc, argv);
302ab10d69eSMasami Hiramatsu }
303ab10d69eSMasami Hiramatsu
trace_uprobe_match(const char * system,const char * event,int argc,const char ** argv,struct dyn_event * ev)304ab10d69eSMasami Hiramatsu static bool trace_uprobe_match(const char *system, const char *event,
305ab10d69eSMasami Hiramatsu int argc, const char **argv, struct dyn_event *ev)
306ab10d69eSMasami Hiramatsu {
307ab10d69eSMasami Hiramatsu struct trace_uprobe *tu = to_trace_uprobe(ev);
308ab10d69eSMasami Hiramatsu
3090597c49cSMasami Hiramatsu return (event[0] == '\0' ||
31030199137SMasami Hiramatsu strcmp(trace_probe_name(&tu->tp), event) == 0) &&
3110597c49cSMasami Hiramatsu (!system || strcmp(trace_probe_group_name(&tu->tp), system) == 0) &&
3120597c49cSMasami Hiramatsu trace_uprobe_match_command_head(tu, argc, argv);
3130597c49cSMasami Hiramatsu }
31495c104c3SLinyu Yuan
31595c104c3SLinyu Yuan static nokprobe_inline struct trace_uprobe *
trace_uprobe_primary_from_call(struct trace_event_call * call)316ab10d69eSMasami Hiramatsu trace_uprobe_primary_from_call(struct trace_event_call *call)
317ab10d69eSMasami Hiramatsu {
3180597c49cSMasami Hiramatsu struct trace_probe *tp;
3190597c49cSMasami Hiramatsu
32060d53e2cSMasami Hiramatsu tp = trace_probe_primary_from_call(call);
32160d53e2cSMasami Hiramatsu if (WARN_ON_ONCE(!tp))
32260d53e2cSMasami Hiramatsu return NULL;
32360d53e2cSMasami Hiramatsu
32460d53e2cSMasami Hiramatsu return container_of(tp, struct trace_uprobe, tp);
32560d53e2cSMasami Hiramatsu }
32660d53e2cSMasami Hiramatsu
32760d53e2cSMasami Hiramatsu /*
32860d53e2cSMasami Hiramatsu * Allocate new trace_uprobe and initialize it (including uprobes).
32960d53e2cSMasami Hiramatsu */
33060d53e2cSMasami Hiramatsu static struct trace_uprobe *
alloc_trace_uprobe(const char * group,const char * event,int nargs,bool is_ret)33160d53e2cSMasami Hiramatsu alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)
332f3f096cfSSrikar Dronamraju {
333f3f096cfSSrikar Dronamraju struct trace_uprobe *tu;
334f3f096cfSSrikar Dronamraju int ret;
335f3f096cfSSrikar Dronamraju
336c1ae5c75SOleg Nesterov tu = kzalloc(struct_size(tu, tp.args, nargs), GFP_KERNEL);
337f3f096cfSSrikar Dronamraju if (!tu)
338f3f096cfSSrikar Dronamraju return ERR_PTR(-ENOMEM);
339455b2899SMasami Hiramatsu
340f3f096cfSSrikar Dronamraju ret = trace_probe_init(&tu->tp, event, group, true);
341845cbf3eSSteven Rostedt (VMware) if (ret < 0)
342f3f096cfSSrikar Dronamraju goto error;
343f3f096cfSSrikar Dronamraju
344f3f096cfSSrikar Dronamraju dyn_event_init(&tu->devent, &trace_uprobe_ops);
345b61387cbSMasami Hiramatsu tu->consumer.handler = uprobe_dispatcher;
346455b2899SMasami Hiramatsu if (is_ret)
347f3f096cfSSrikar Dronamraju tu->consumer.ret_handler = uretprobe_dispatcher;
348f3f096cfSSrikar Dronamraju init_trace_uprobe_filter(tu->tp.event->filter);
3490597c49cSMasami Hiramatsu return tu;
350a932b738SOleg Nesterov
351c1ae5c75SOleg Nesterov error:
352c1ae5c75SOleg Nesterov kfree(tu);
353b61387cbSMasami Hiramatsu
354f3f096cfSSrikar Dronamraju return ERR_PTR(ret);
355f3f096cfSSrikar Dronamraju }
356f3f096cfSSrikar Dronamraju
free_trace_uprobe(struct trace_uprobe * tu)357f3f096cfSSrikar Dronamraju static void free_trace_uprobe(struct trace_uprobe *tu)
358f3f096cfSSrikar Dronamraju {
359455b2899SMasami Hiramatsu if (!tu)
360f3f096cfSSrikar Dronamraju return;
361f3f096cfSSrikar Dronamraju
362f3f096cfSSrikar Dronamraju path_put(&tu->path);
363f3f096cfSSrikar Dronamraju trace_probe_cleanup(&tu->tp);
3640597c49cSMasami Hiramatsu kfree(tu->filename);
3650597c49cSMasami Hiramatsu kfree(tu);
3660597c49cSMasami Hiramatsu }
3670c92c7a3SSong Liu
find_probe_event(const char * event,const char * group)368455b2899SMasami Hiramatsu static struct trace_uprobe *find_probe_event(const char *event, const char *group)
369f3f096cfSSrikar Dronamraju {
370f3f096cfSSrikar Dronamraju struct dyn_event *pos;
371f3f096cfSSrikar Dronamraju struct trace_uprobe *tu;
372f3f096cfSSrikar Dronamraju
373f3f096cfSSrikar Dronamraju for_each_trace_uprobe(tu, pos)
374f3f096cfSSrikar Dronamraju if (strcmp(trace_probe_name(&tu->tp), event) == 0 &&
3750597c49cSMasami Hiramatsu strcmp(trace_probe_group_name(&tu->tp), group) == 0)
376f3f096cfSSrikar Dronamraju return tu;
377f3f096cfSSrikar Dronamraju
3780597c49cSMasami Hiramatsu return NULL;
379b55ce203SMasami Hiramatsu }
380b55ce203SMasami Hiramatsu
381f3f096cfSSrikar Dronamraju /* Unregister a trace_uprobe and probe_event */
unregister_trace_uprobe(struct trace_uprobe * tu)382f3f096cfSSrikar Dronamraju static int unregister_trace_uprobe(struct trace_uprobe *tu)
383f3f096cfSSrikar Dronamraju {
384f3f096cfSSrikar Dronamraju int ret;
385f3f096cfSSrikar Dronamraju
3860597c49cSMasami Hiramatsu if (trace_probe_has_sibling(&tu->tp))
387c6c2401dSSteven Rostedt (Red Hat) goto unreg;
388f3f096cfSSrikar Dronamraju
389c6c2401dSSteven Rostedt (Red Hat) /* If there's a reference to the dynamic event */
390c6c2401dSSteven Rostedt (Red Hat) if (trace_event_dyn_busy(trace_probe_event_call(&tu->tp)))
39141af3cf5SMasami Hiramatsu return -EBUSY;
39241af3cf5SMasami Hiramatsu
39341af3cf5SMasami Hiramatsu ret = unregister_uprobe_event(tu);
3941d18538eSSteven Rostedt (VMware) if (ret)
3951d18538eSSteven Rostedt (VMware) return ret;
3961d18538eSSteven Rostedt (VMware)
3971d18538eSSteven Rostedt (VMware) unreg:
398c6c2401dSSteven Rostedt (Red Hat) dyn_event_remove(&tu->devent);
399c6c2401dSSteven Rostedt (Red Hat) trace_probe_unlink(&tu->tp);
400c6c2401dSSteven Rostedt (Red Hat) free_trace_uprobe(tu);
401c6c2401dSSteven Rostedt (Red Hat) return 0;
40241af3cf5SMasami Hiramatsu }
4030597c49cSMasami Hiramatsu
trace_uprobe_has_same_uprobe(struct trace_uprobe * orig,struct trace_uprobe * comp)40441af3cf5SMasami Hiramatsu static bool trace_uprobe_has_same_uprobe(struct trace_uprobe *orig,
405f3f096cfSSrikar Dronamraju struct trace_uprobe *comp)
406c6c2401dSSteven Rostedt (Red Hat) {
407f3f096cfSSrikar Dronamraju struct trace_probe_event *tpe = orig->tp.event;
408f3f096cfSSrikar Dronamraju struct inode *comp_inode = d_real_inode(comp->path.dentry);
409fe60b0ceSMasami Hiramatsu int i;
410fe60b0ceSMasami Hiramatsu
411fe60b0ceSMasami Hiramatsu list_for_each_entry(orig, &tpe->probes, tp.list) {
412fe60b0ceSMasami Hiramatsu if (comp_inode != d_real_inode(orig->path.dentry) ||
413fe60b0ceSMasami Hiramatsu comp->offset != orig->offset)
414fe60b0ceSMasami Hiramatsu continue;
415fe60b0ceSMasami Hiramatsu
416e161c6bfSJiri Olsa /*
417fe60b0ceSMasami Hiramatsu * trace_probe_compare_arg_type() ensured that nr_args and
418fe60b0ceSMasami Hiramatsu * each argument name and type are same. Let's compare comm.
419fe60b0ceSMasami Hiramatsu */
420fe60b0ceSMasami Hiramatsu for (i = 0; i < orig->tp.nr_args; i++) {
421fe60b0ceSMasami Hiramatsu if (strcmp(orig->tp.args[i].comm,
422fe60b0ceSMasami Hiramatsu comp->tp.args[i].comm))
423fe60b0ceSMasami Hiramatsu break;
424fe60b0ceSMasami Hiramatsu }
425fe60b0ceSMasami Hiramatsu
426fe60b0ceSMasami Hiramatsu if (i == orig->tp.nr_args)
427fe60b0ceSMasami Hiramatsu return true;
428f8d7ab2bSSrikar Dronamraju }
429fe60b0ceSMasami Hiramatsu
430fe60b0ceSMasami Hiramatsu return false;
431f8d7ab2bSSrikar Dronamraju }
432fe60b0ceSMasami Hiramatsu
append_trace_uprobe(struct trace_uprobe * tu,struct trace_uprobe * to)433fe60b0ceSMasami Hiramatsu static int append_trace_uprobe(struct trace_uprobe *tu, struct trace_uprobe *to)
434fe60b0ceSMasami Hiramatsu {
435fe60b0ceSMasami Hiramatsu int ret;
436fe60b0ceSMasami Hiramatsu
437fe60b0ceSMasami Hiramatsu ret = trace_probe_compare_arg_type(&tu->tp, &to->tp);
43841af3cf5SMasami Hiramatsu if (ret) {
43941af3cf5SMasami Hiramatsu /* Note that argument starts index = 2 */
44041af3cf5SMasami Hiramatsu trace_probe_log_set_index(ret + 1);
44141af3cf5SMasami Hiramatsu trace_probe_log_err(0, DIFF_ARG_TYPE);
442fe60b0ceSMasami Hiramatsu return -EEXIST;
443fe60b0ceSMasami Hiramatsu }
444fe60b0ceSMasami Hiramatsu if (trace_uprobe_has_same_uprobe(to, tu)) {
445fe60b0ceSMasami Hiramatsu trace_probe_log_set_index(0);
446fe60b0ceSMasami Hiramatsu trace_probe_log_err(0, SAME_PROBE);
447fe60b0ceSMasami Hiramatsu return -EEXIST;
448fe60b0ceSMasami Hiramatsu }
449fe60b0ceSMasami Hiramatsu
450fe60b0ceSMasami Hiramatsu /* Append to existing event */
451fe60b0ceSMasami Hiramatsu ret = trace_probe_append(&tu->tp, &to->tp);
452fe60b0ceSMasami Hiramatsu if (!ret)
453fe60b0ceSMasami Hiramatsu dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp));
454fe60b0ceSMasami Hiramatsu
45541af3cf5SMasami Hiramatsu return ret;
45641af3cf5SMasami Hiramatsu }
45741af3cf5SMasami Hiramatsu
4588b0e6c74SSteven Rostedt (VMware) /*
45941af3cf5SMasami Hiramatsu * Uprobe with multiple reference counter is not allowed. i.e.
46041af3cf5SMasami Hiramatsu * If inode and offset matches, reference counter offset *must*
46141af3cf5SMasami Hiramatsu * match as well. Though, there is one exception: If user is
46241af3cf5SMasami Hiramatsu * replacing old trace_uprobe with new one(same group/event),
463ccea8727SRavi Bangoria * then we allow same uprobe with new reference counter as far
464ccea8727SRavi Bangoria * as the new one does not conflict with any other existing
465ccea8727SRavi Bangoria * ones.
466ccea8727SRavi Bangoria */
validate_ref_ctr_offset(struct trace_uprobe * new)467ccea8727SRavi Bangoria static int validate_ref_ctr_offset(struct trace_uprobe *new)
468ccea8727SRavi Bangoria {
469ccea8727SRavi Bangoria struct dyn_event *pos;
470ccea8727SRavi Bangoria struct trace_uprobe *tmp;
471ccea8727SRavi Bangoria struct inode *new_inode = d_real_inode(new->path.dentry);
47241af3cf5SMasami Hiramatsu
473ccea8727SRavi Bangoria for_each_trace_uprobe(tmp, pos) {
4740597c49cSMasami Hiramatsu if (new_inode == d_real_inode(tmp->path.dentry) &&
47541af3cf5SMasami Hiramatsu new->offset == tmp->offset &&
476ccea8727SRavi Bangoria new->ref_ctr_offset != tmp->ref_ctr_offset) {
477ccea8727SRavi Bangoria pr_warn("Reference counter offset mismatch.");
4780597c49cSMasami Hiramatsu return -EINVAL;
47941af3cf5SMasami Hiramatsu }
480ccea8727SRavi Bangoria }
481ccea8727SRavi Bangoria return 0;
482ccea8727SRavi Bangoria }
48341af3cf5SMasami Hiramatsu
484ccea8727SRavi Bangoria /* Register a trace_uprobe and probe_event */
register_trace_uprobe(struct trace_uprobe * tu)485ccea8727SRavi Bangoria static int register_trace_uprobe(struct trace_uprobe *tu)
48641af3cf5SMasami Hiramatsu {
487ccea8727SRavi Bangoria struct trace_uprobe *old_tu;
488ccea8727SRavi Bangoria int ret;
489f3f096cfSSrikar Dronamraju
490f3f096cfSSrikar Dronamraju mutex_lock(&event_mutex);
491f3f096cfSSrikar Dronamraju
49214577c39SNamhyung Kim ret = validate_ref_ctr_offset(tu);
493f3f096cfSSrikar Dronamraju if (ret)
494f3f096cfSSrikar Dronamraju goto end;
4950597c49cSMasami Hiramatsu
496f3f096cfSSrikar Dronamraju /* register as an event */
49741af3cf5SMasami Hiramatsu old_tu = find_probe_event(trace_probe_name(&tu->tp),
498c6c2401dSSteven Rostedt (Red Hat) trace_probe_group_name(&tu->tp));
499c6c2401dSSteven Rostedt (Red Hat) if (old_tu) {
50041af3cf5SMasami Hiramatsu if (is_ret_probe(tu) != is_ret_probe(old_tu)) {
50141af3cf5SMasami Hiramatsu trace_probe_log_set_index(0);
50241af3cf5SMasami Hiramatsu trace_probe_log_err(0, DIFF_PROBE_TYPE);
50341af3cf5SMasami Hiramatsu ret = -EEXIST;
50441af3cf5SMasami Hiramatsu } else {
50541af3cf5SMasami Hiramatsu ret = append_trace_uprobe(tu, old_tu);
50641af3cf5SMasami Hiramatsu }
50741af3cf5SMasami Hiramatsu goto end;
50841af3cf5SMasami Hiramatsu }
50941af3cf5SMasami Hiramatsu
51041af3cf5SMasami Hiramatsu ret = register_uprobe_event(tu);
51141af3cf5SMasami Hiramatsu if (ret) {
51241af3cf5SMasami Hiramatsu if (ret == -EEXIST) {
513c6c2401dSSteven Rostedt (Red Hat) trace_probe_log_set_index(0);
514f3f096cfSSrikar Dronamraju trace_probe_log_err(0, EVENT_EXIST);
515f3f096cfSSrikar Dronamraju } else
516f3f096cfSSrikar Dronamraju pr_warn("Failed to register probe event(%d)\n", ret);
5178e242060SMasami Hiramatsu goto end;
5188e242060SMasami Hiramatsu }
5198e242060SMasami Hiramatsu
5208e242060SMasami Hiramatsu dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp));
521a395d6a7SJoe Perches
522f3f096cfSSrikar Dronamraju end:
523f3f096cfSSrikar Dronamraju mutex_unlock(&event_mutex);
524f3f096cfSSrikar Dronamraju
5258b0e6c74SSteven Rostedt (VMware) return ret;
526f3f096cfSSrikar Dronamraju }
527f3f096cfSSrikar Dronamraju
5280597c49cSMasami Hiramatsu /*
529f3f096cfSSrikar Dronamraju * Argument syntax:
530f3f096cfSSrikar Dronamraju * - Add uprobe: p|r[:[GRP/][EVENT]] PATH:OFFSET[%return][(REF)] [FETCHARGS]
531f3f096cfSSrikar Dronamraju */
__trace_uprobe_create(int argc,const char ** argv)532f3f096cfSSrikar Dronamraju static int __trace_uprobe_create(int argc, const char **argv)
533f3f096cfSSrikar Dronamraju {
534f3f096cfSSrikar Dronamraju struct trace_uprobe *tu;
53595c104c3SLinyu Yuan const char *event = NULL, *group = UPROBE_EVENT_SYSTEM;
536f3f096cfSSrikar Dronamraju char *arg, *filename, *rctr, *rctr_end, *tmp;
537d262271dSMasami Hiramatsu char buf[MAX_EVENT_NAME_LEN];
538f3f096cfSSrikar Dronamraju char gbuf[MAX_EVENT_NAME_LEN];
539f3f096cfSSrikar Dronamraju enum probe_print_type ptype;
5400597c49cSMasami Hiramatsu struct path path;
5410597c49cSMasami Hiramatsu unsigned long offset, ref_ctr_offset;
542f3f096cfSSrikar Dronamraju bool is_return = false;
54395c104c3SLinyu Yuan int i, ret;
544007517a0SSteven Rostedt (VMware)
545f3f096cfSSrikar Dronamraju ref_ctr_offset = 0;
5461cc33161SRavi Bangoria
5470597c49cSMasami Hiramatsu switch (argv[0][0]) {
548f3f096cfSSrikar Dronamraju case 'r':
549f3f096cfSSrikar Dronamraju is_return = true;
5501cc33161SRavi Bangoria break;
551f3f096cfSSrikar Dronamraju case 'p':
552f01098c7SEiichi Tsukata break;
553f01098c7SEiichi Tsukata default:
5544ee5a52eSOleg Nesterov return -ECANCELED;
555f01098c7SEiichi Tsukata }
556f01098c7SEiichi Tsukata
557f01098c7SEiichi Tsukata if (argc < 2)
558f01098c7SEiichi Tsukata return -ECANCELED;
559f01098c7SEiichi Tsukata
560f01098c7SEiichi Tsukata if (argv[0][1] == ':')
561f01098c7SEiichi Tsukata event = &argv[0][2];
562f01098c7SEiichi Tsukata
5630597c49cSMasami Hiramatsu if (!strchr(argv[1], '/'))
564f3f096cfSSrikar Dronamraju return -ECANCELED;
5650597c49cSMasami Hiramatsu
566f3f096cfSSrikar Dronamraju filename = kstrdup(argv[1], GFP_KERNEL);
567f3f096cfSSrikar Dronamraju if (!filename)
5680597c49cSMasami Hiramatsu return -ENOMEM;
5690597c49cSMasami Hiramatsu
570f3f096cfSSrikar Dronamraju /* Find the last occurrence, in case the path contains ':' too. */
5710597c49cSMasami Hiramatsu arg = strrchr(filename, ':');
5720597c49cSMasami Hiramatsu if (!arg || !isdigit(arg[1])) {
5730597c49cSMasami Hiramatsu kfree(filename);
574f3f096cfSSrikar Dronamraju return -ECANCELED;
5756496bb72SKenny Yu }
5760597c49cSMasami Hiramatsu
5770597c49cSMasami Hiramatsu trace_probe_log_init("trace_uprobe", argc, argv);
5780597c49cSMasami Hiramatsu trace_probe_log_set_index(1); /* filename is the 2nd argument */
5790597c49cSMasami Hiramatsu
5800597c49cSMasami Hiramatsu *arg++ = '\0';
581f3f096cfSSrikar Dronamraju ret = kern_path(filename, LOOKUP_FOLLOW, &path);
582ab105a4fSMasami Hiramatsu if (ret) {
583ab105a4fSMasami Hiramatsu trace_probe_log_err(0, FILE_NOT_FOUND);
584ab105a4fSMasami Hiramatsu kfree(filename);
585f3f096cfSSrikar Dronamraju trace_probe_log_clear();
586f3f096cfSSrikar Dronamraju return ret;
5870597c49cSMasami Hiramatsu }
588ab105a4fSMasami Hiramatsu if (!d_is_reg(path.dentry)) {
5890597c49cSMasami Hiramatsu trace_probe_log_err(0, NO_REGULAR_FILE);
590ab105a4fSMasami Hiramatsu ret = -EINVAL;
5910c92c7a3SSong Liu goto fail_address_parse;
5920597c49cSMasami Hiramatsu }
5930c92c7a3SSong Liu
594ab105a4fSMasami Hiramatsu /* Parse reference counter offset if specified. */
595d24d7dbfSJovi Zhang rctr = strchr(arg, '(');
596d24d7dbfSJovi Zhang if (rctr) {
597d24d7dbfSJovi Zhang rctr_end = strchr(rctr, ')');
598f3f096cfSSrikar Dronamraju if (!rctr_end) {
5991cc33161SRavi Bangoria ret = -EINVAL;
6001cc33161SRavi Bangoria rctr_end = rctr + strlen(rctr);
6011cc33161SRavi Bangoria trace_probe_log_err(rctr_end - filename,
6021cc33161SRavi Bangoria REFCNT_OPEN_BRACE);
603ab105a4fSMasami Hiramatsu goto fail_address_parse;
6041cc33161SRavi Bangoria } else if (rctr_end[1] != '\0') {
605ab105a4fSMasami Hiramatsu ret = -EINVAL;
606ab105a4fSMasami Hiramatsu trace_probe_log_err(rctr_end + 1 - filename,
607ab105a4fSMasami Hiramatsu BAD_REFCNT_SUFFIX);
608ab105a4fSMasami Hiramatsu goto fail_address_parse;
609ab105a4fSMasami Hiramatsu }
610ab105a4fSMasami Hiramatsu
611ab105a4fSMasami Hiramatsu *rctr++ = '\0';
612ab105a4fSMasami Hiramatsu *rctr_end = '\0';
6131cc33161SRavi Bangoria ret = kstrtoul(rctr, 0, &ref_ctr_offset);
6141cc33161SRavi Bangoria if (ret) {
6151cc33161SRavi Bangoria trace_probe_log_err(rctr - filename, BAD_REFCNT);
6161cc33161SRavi Bangoria goto fail_address_parse;
6171cc33161SRavi Bangoria }
6181cc33161SRavi Bangoria }
6191cc33161SRavi Bangoria
620ab105a4fSMasami Hiramatsu /* Check if there is %return suffix */
6211cc33161SRavi Bangoria tmp = strchr(arg, '%');
6221cc33161SRavi Bangoria if (tmp) {
6231cc33161SRavi Bangoria if (!strcmp(tmp, "%return")) {
6241cc33161SRavi Bangoria *tmp = '\0';
6253dd3aae3SMasami Hiramatsu is_return = true;
6263dd3aae3SMasami Hiramatsu } else {
6273dd3aae3SMasami Hiramatsu trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX);
6283dd3aae3SMasami Hiramatsu ret = -EINVAL;
6293dd3aae3SMasami Hiramatsu goto fail_address_parse;
6303dd3aae3SMasami Hiramatsu }
6313dd3aae3SMasami Hiramatsu }
6323dd3aae3SMasami Hiramatsu
6333dd3aae3SMasami Hiramatsu /* Parse uprobe offset. */
6343dd3aae3SMasami Hiramatsu ret = kstrtoul(arg, 0, &offset);
6353dd3aae3SMasami Hiramatsu if (ret) {
6363dd3aae3SMasami Hiramatsu trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS);
6373dd3aae3SMasami Hiramatsu goto fail_address_parse;
6381cc33161SRavi Bangoria }
63984d7ed79SOleg Nesterov
640ab105a4fSMasami Hiramatsu /* setup a probe */
641ab105a4fSMasami Hiramatsu trace_probe_log_set_index(0);
64284d7ed79SOleg Nesterov if (event) {
643ab105a4fSMasami Hiramatsu ret = traceprobe_parse_event_name(&event, &group, gbuf,
644f3f096cfSSrikar Dronamraju event - argv[0]);
645f3f096cfSSrikar Dronamraju if (ret)
646ab105a4fSMasami Hiramatsu goto fail_address_parse;
6470597c49cSMasami Hiramatsu }
64895c104c3SLinyu Yuan
649ab105a4fSMasami Hiramatsu if (!event) {
6500597c49cSMasami Hiramatsu char *tail;
6510597c49cSMasami Hiramatsu char *ptr;
65295c104c3SLinyu Yuan
65395c104c3SLinyu Yuan tail = kstrdup(kbasename(filename), GFP_KERNEL);
65495c104c3SLinyu Yuan if (!tail) {
655b2e902f0SAndy Shevchenko ret = -ENOMEM;
656f3f096cfSSrikar Dronamraju goto fail_address_parse;
657f3f096cfSSrikar Dronamraju }
658b2e902f0SAndy Shevchenko
659b2e902f0SAndy Shevchenko ptr = strpbrk(tail, ".-_");
660f3f096cfSSrikar Dronamraju if (ptr)
661f3f096cfSSrikar Dronamraju *ptr = '\0';
662f3f096cfSSrikar Dronamraju
663f3f096cfSSrikar Dronamraju snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset);
664f3f096cfSSrikar Dronamraju event = buf;
665f3f096cfSSrikar Dronamraju kfree(tail);
666f3f096cfSSrikar Dronamraju }
667f3f096cfSSrikar Dronamraju
668f3f096cfSSrikar Dronamraju argc -= 2;
669f3f096cfSSrikar Dronamraju argv += 2;
670f3f096cfSSrikar Dronamraju
671f3f096cfSSrikar Dronamraju tu = alloc_trace_uprobe(group, event, argc, is_return);
672f3f096cfSSrikar Dronamraju if (IS_ERR(tu)) {
673ab105a4fSMasami Hiramatsu ret = PTR_ERR(tu);
674ab105a4fSMasami Hiramatsu /* This must return -ENOMEM otherwise there is a bug */
675ab105a4fSMasami Hiramatsu WARN_ON_ONCE(ret != -ENOMEM);
6764ee5a52eSOleg Nesterov goto fail_address_parse;
677f3f096cfSSrikar Dronamraju }
678f3f096cfSSrikar Dronamraju tu->offset = offset;
679a039480eSMasami Hiramatsu tu->ref_ctr_offset = ref_ctr_offset;
680a039480eSMasami Hiramatsu tu->path = path;
681f3f096cfSSrikar Dronamraju tu->filename = filename;
682f3f096cfSSrikar Dronamraju
683f3f096cfSSrikar Dronamraju /* parse arguments */
6841cc33161SRavi Bangoria for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
6850c92c7a3SSong Liu struct traceprobe_parse_context ctx = {
6860597c49cSMasami Hiramatsu .flags = (is_return ? TPARG_FL_RETURN : 0) | TPARG_FL_USER,
687f3f096cfSSrikar Dronamraju };
6880597c49cSMasami Hiramatsu
6890597c49cSMasami Hiramatsu trace_probe_log_set_index(i + 2);
6901b8b0cd7SMasami Hiramatsu (Google) ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i], &ctx);
6911b8b0cd7SMasami Hiramatsu (Google) traceprobe_finish_parse(&ctx);
6921b8b0cd7SMasami Hiramatsu (Google) if (ret)
6931b8b0cd7SMasami Hiramatsu (Google) goto error;
694ab105a4fSMasami Hiramatsu }
6951b8b0cd7SMasami Hiramatsu (Google)
696*b1d1e904SMasami Hiramatsu (Google) ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
697d00bbea9SMasami Hiramatsu ret = traceprobe_set_print_fmt(&tu->tp, ptype);
698f3f096cfSSrikar Dronamraju if (ret < 0)
699f3f096cfSSrikar Dronamraju goto error;
700f3f096cfSSrikar Dronamraju
701007517a0SSteven Rostedt (VMware) ret = register_trace_uprobe(tu);
702007517a0SSteven Rostedt (VMware) if (!ret)
703b4d4b96bSMasami Hiramatsu goto out;
704b4d4b96bSMasami Hiramatsu
705b4d4b96bSMasami Hiramatsu error:
706f3f096cfSSrikar Dronamraju free_trace_uprobe(tu);
707ab105a4fSMasami Hiramatsu out:
708ab105a4fSMasami Hiramatsu trace_probe_log_clear();
709f3f096cfSSrikar Dronamraju return ret;
710f3f096cfSSrikar Dronamraju
711f3f096cfSSrikar Dronamraju fail_address_parse:
712ab105a4fSMasami Hiramatsu trace_probe_log_clear();
713ab105a4fSMasami Hiramatsu path_put(&path);
714f3f096cfSSrikar Dronamraju kfree(filename);
715f3f096cfSSrikar Dronamraju
716f3f096cfSSrikar Dronamraju return ret;
717ab105a4fSMasami Hiramatsu }
7180c92c7a3SSong Liu
trace_uprobe_create(const char * raw_command)7190597c49cSMasami Hiramatsu int trace_uprobe_create(const char *raw_command)
720f3f096cfSSrikar Dronamraju {
721f3f096cfSSrikar Dronamraju return trace_probe_create(raw_command, __trace_uprobe_create);
722f3f096cfSSrikar Dronamraju }
723f3f096cfSSrikar Dronamraju
create_or_delete_trace_uprobe(const char * raw_command)724d262271dSMasami Hiramatsu static int create_or_delete_trace_uprobe(const char *raw_command)
725d262271dSMasami Hiramatsu {
726d262271dSMasami Hiramatsu int ret;
727d262271dSMasami Hiramatsu
728d262271dSMasami Hiramatsu if (raw_command[0] == '-')
729d262271dSMasami Hiramatsu return dyn_event_release(raw_command, &trace_uprobe_ops);
730f3f096cfSSrikar Dronamraju
7310597c49cSMasami Hiramatsu ret = trace_uprobe_create(raw_command);
732f3f096cfSSrikar Dronamraju return ret == -ECANCELED ? -EINVAL : ret;
733d262271dSMasami Hiramatsu }
734d262271dSMasami Hiramatsu
trace_uprobe_release(struct dyn_event * ev)7350597c49cSMasami Hiramatsu static int trace_uprobe_release(struct dyn_event *ev)
736d262271dSMasami Hiramatsu {
7370597c49cSMasami Hiramatsu struct trace_uprobe *tu = to_trace_uprobe(ev);
738547cd9eaSMasami Hiramatsu
7390597c49cSMasami Hiramatsu return unregister_trace_uprobe(tu);
7400597c49cSMasami Hiramatsu }
7410597c49cSMasami Hiramatsu
7420597c49cSMasami Hiramatsu /* Probes listing interfaces */
trace_uprobe_show(struct seq_file * m,struct dyn_event * ev)7430597c49cSMasami Hiramatsu static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev)
7440597c49cSMasami Hiramatsu {
745f3f096cfSSrikar Dronamraju struct trace_uprobe *tu = to_trace_uprobe(ev);
746f3f096cfSSrikar Dronamraju char c = is_ret_probe(tu) ? 'r' : 'p';
747f3f096cfSSrikar Dronamraju int i;
7480597c49cSMasami Hiramatsu
749f3f096cfSSrikar Dronamraju seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, trace_probe_group_name(&tu->tp),
7500597c49cSMasami Hiramatsu trace_probe_name(&tu->tp), tu->filename,
7513ede82ddSOleg Nesterov (int)(sizeof(void *) * 2), tu->offset);
752f3f096cfSSrikar Dronamraju
753f3f096cfSSrikar Dronamraju if (tu->ref_ctr_offset)
754b55ce203SMasami Hiramatsu seq_printf(m, "(0x%lx)", tu->ref_ctr_offset);
755b55ce203SMasami Hiramatsu
756a64b2c01SRavi Bangoria for (i = 0; i < tu->tp.nr_args; i++)
757f3f096cfSSrikar Dronamraju seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
7581cc33161SRavi Bangoria
7591cc33161SRavi Bangoria seq_putc(m, '\n');
7601cc33161SRavi Bangoria return 0;
76114577c39SNamhyung Kim }
76214577c39SNamhyung Kim
probes_seq_show(struct seq_file * m,void * v)763f3f096cfSSrikar Dronamraju static int probes_seq_show(struct seq_file *m, void *v)
764fa6f0cc7SRasmus Villemoes {
765f3f096cfSSrikar Dronamraju struct dyn_event *ev = v;
766f3f096cfSSrikar Dronamraju
767f3f096cfSSrikar Dronamraju if (!is_trace_uprobe(ev))
7680597c49cSMasami Hiramatsu return 0;
7690597c49cSMasami Hiramatsu
7700597c49cSMasami Hiramatsu return trace_uprobe_show(m, ev);
7710597c49cSMasami Hiramatsu }
7720597c49cSMasami Hiramatsu
7730597c49cSMasami Hiramatsu static const struct seq_operations probes_seq_op = {
7740597c49cSMasami Hiramatsu .start = dyn_event_seq_start,
7750597c49cSMasami Hiramatsu .next = dyn_event_seq_next,
7760597c49cSMasami Hiramatsu .stop = dyn_event_seq_stop,
7770597c49cSMasami Hiramatsu .show = probes_seq_show
778f3f096cfSSrikar Dronamraju };
7790597c49cSMasami Hiramatsu
probes_open(struct inode * inode,struct file * file)7800597c49cSMasami Hiramatsu static int probes_open(struct inode *inode, struct file *file)
7810597c49cSMasami Hiramatsu {
782f3f096cfSSrikar Dronamraju int ret;
783f3f096cfSSrikar Dronamraju
784f3f096cfSSrikar Dronamraju ret = security_locked_down(LOCKDOWN_TRACEFS);
785f3f096cfSSrikar Dronamraju if (ret)
786f3f096cfSSrikar Dronamraju return ret;
787c6c2401dSSteven Rostedt (Red Hat)
788c6c2401dSSteven Rostedt (Red Hat) if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
78917911ff3SSteven Rostedt (VMware) ret = dyn_events_release_all(&trace_uprobe_ops);
79017911ff3SSteven Rostedt (VMware) if (ret)
79117911ff3SSteven Rostedt (VMware) return ret;
79217911ff3SSteven Rostedt (VMware) }
793c6c2401dSSteven Rostedt (Red Hat)
7940597c49cSMasami Hiramatsu return seq_open(file, &probes_seq_op);
795c6c2401dSSteven Rostedt (Red Hat) }
796c6c2401dSSteven Rostedt (Red Hat)
probes_write(struct file * file,const char __user * buffer,size_t count,loff_t * ppos)797c6c2401dSSteven Rostedt (Red Hat) static ssize_t probes_write(struct file *file, const char __user *buffer,
798f3f096cfSSrikar Dronamraju size_t count, loff_t *ppos)
799f3f096cfSSrikar Dronamraju {
800f3f096cfSSrikar Dronamraju return trace_parse_run_command(file, buffer, count, ppos,
801f3f096cfSSrikar Dronamraju create_or_delete_trace_uprobe);
802f3f096cfSSrikar Dronamraju }
803f3f096cfSSrikar Dronamraju
804f3f096cfSSrikar Dronamraju static const struct file_operations uprobe_events_ops = {
8050597c49cSMasami Hiramatsu .owner = THIS_MODULE,
8060597c49cSMasami Hiramatsu .open = probes_open,
807f3f096cfSSrikar Dronamraju .read = seq_read,
808f3f096cfSSrikar Dronamraju .llseek = seq_lseek,
809f3f096cfSSrikar Dronamraju .release = seq_release,
810f3f096cfSSrikar Dronamraju .write = probes_write,
811f3f096cfSSrikar Dronamraju };
812f3f096cfSSrikar Dronamraju
813f3f096cfSSrikar Dronamraju /* Probes profiling interfaces */
probes_profile_seq_show(struct seq_file * m,void * v)814f3f096cfSSrikar Dronamraju static int probes_profile_seq_show(struct seq_file *m, void *v)
815f3f096cfSSrikar Dronamraju {
816f3f096cfSSrikar Dronamraju struct dyn_event *ev = v;
817f3f096cfSSrikar Dronamraju struct trace_uprobe *tu;
818f3f096cfSSrikar Dronamraju
819f3f096cfSSrikar Dronamraju if (!is_trace_uprobe(ev))
820f3f096cfSSrikar Dronamraju return 0;
8210597c49cSMasami Hiramatsu
8220597c49cSMasami Hiramatsu tu = to_trace_uprobe(ev);
823f3f096cfSSrikar Dronamraju seq_printf(m, " %s %-44s %15lu\n", tu->filename,
8240597c49cSMasami Hiramatsu trace_probe_name(&tu->tp), tu->nhit);
8250597c49cSMasami Hiramatsu return 0;
8260597c49cSMasami Hiramatsu }
8270597c49cSMasami Hiramatsu
828de7b2973SMathieu Desnoyers static const struct seq_operations profile_seq_op = {
829b55ce203SMasami Hiramatsu .start = dyn_event_seq_start,
830f3f096cfSSrikar Dronamraju .next = dyn_event_seq_next,
831f3f096cfSSrikar Dronamraju .stop = dyn_event_seq_stop,
832f3f096cfSSrikar Dronamraju .show = probes_profile_seq_show
833f3f096cfSSrikar Dronamraju };
8340597c49cSMasami Hiramatsu
profile_open(struct inode * inode,struct file * file)8350597c49cSMasami Hiramatsu static int profile_open(struct inode *inode, struct file *file)
8360597c49cSMasami Hiramatsu {
837f3f096cfSSrikar Dronamraju int ret;
838f3f096cfSSrikar Dronamraju
839f3f096cfSSrikar Dronamraju ret = security_locked_down(LOCKDOWN_TRACEFS);
840f3f096cfSSrikar Dronamraju if (ret)
841f3f096cfSSrikar Dronamraju return ret;
84217911ff3SSteven Rostedt (VMware)
84317911ff3SSteven Rostedt (VMware) return seq_open(file, &profile_seq_op);
84417911ff3SSteven Rostedt (VMware) }
84517911ff3SSteven Rostedt (VMware)
84617911ff3SSteven Rostedt (VMware) static const struct file_operations uprobe_profile_ops = {
84717911ff3SSteven Rostedt (VMware) .owner = THIS_MODULE,
848f3f096cfSSrikar Dronamraju .open = profile_open,
849f3f096cfSSrikar Dronamraju .read = seq_read,
850f3f096cfSSrikar Dronamraju .llseek = seq_lseek,
851f3f096cfSSrikar Dronamraju .release = seq_release,
852f3f096cfSSrikar Dronamraju };
853f3f096cfSSrikar Dronamraju
854f3f096cfSSrikar Dronamraju struct uprobe_cpu_buffer {
855f3f096cfSSrikar Dronamraju struct mutex mutex;
856f3f096cfSSrikar Dronamraju void *buf;
857f3f096cfSSrikar Dronamraju };
858f3f096cfSSrikar Dronamraju static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer;
859dcad1a20SNamhyung Kim static int uprobe_buffer_refcnt;
860dcad1a20SNamhyung Kim
uprobe_buffer_init(void)861dcad1a20SNamhyung Kim static int uprobe_buffer_init(void)
862dcad1a20SNamhyung Kim {
863dcad1a20SNamhyung Kim int cpu, err_cpu;
864dcad1a20SNamhyung Kim
865dcad1a20SNamhyung Kim uprobe_cpu_buffer = alloc_percpu(struct uprobe_cpu_buffer);
866dcad1a20SNamhyung Kim if (uprobe_cpu_buffer == NULL)
867dcad1a20SNamhyung Kim return -ENOMEM;
868dcad1a20SNamhyung Kim
869dcad1a20SNamhyung Kim for_each_possible_cpu(cpu) {
870dcad1a20SNamhyung Kim struct page *p = alloc_pages_node(cpu_to_node(cpu),
871dcad1a20SNamhyung Kim GFP_KERNEL, 0);
872dcad1a20SNamhyung Kim if (p == NULL) {
873dcad1a20SNamhyung Kim err_cpu = cpu;
874dcad1a20SNamhyung Kim goto err;
875dcad1a20SNamhyung Kim }
876dcad1a20SNamhyung Kim per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf = page_address(p);
877dcad1a20SNamhyung Kim mutex_init(&per_cpu_ptr(uprobe_cpu_buffer, cpu)->mutex);
878dcad1a20SNamhyung Kim }
879dcad1a20SNamhyung Kim
880dcad1a20SNamhyung Kim return 0;
881dcad1a20SNamhyung Kim
882dcad1a20SNamhyung Kim err:
883dcad1a20SNamhyung Kim for_each_possible_cpu(cpu) {
884dcad1a20SNamhyung Kim if (cpu == err_cpu)
885dcad1a20SNamhyung Kim break;
886dcad1a20SNamhyung Kim free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf);
887dcad1a20SNamhyung Kim }
888dcad1a20SNamhyung Kim
889dcad1a20SNamhyung Kim free_percpu(uprobe_cpu_buffer);
890dcad1a20SNamhyung Kim return -ENOMEM;
891dcad1a20SNamhyung Kim }
892dcad1a20SNamhyung Kim
uprobe_buffer_enable(void)893dcad1a20SNamhyung Kim static int uprobe_buffer_enable(void)
894dcad1a20SNamhyung Kim {
895dcad1a20SNamhyung Kim int ret = 0;
896dcad1a20SNamhyung Kim
897dcad1a20SNamhyung Kim BUG_ON(!mutex_is_locked(&event_mutex));
898dcad1a20SNamhyung Kim
899dcad1a20SNamhyung Kim if (uprobe_buffer_refcnt++ == 0) {
900dcad1a20SNamhyung Kim ret = uprobe_buffer_init();
901dcad1a20SNamhyung Kim if (ret < 0)
902dcad1a20SNamhyung Kim uprobe_buffer_refcnt--;
903dcad1a20SNamhyung Kim }
904dcad1a20SNamhyung Kim
905dcad1a20SNamhyung Kim return ret;
906dcad1a20SNamhyung Kim }
907dcad1a20SNamhyung Kim
uprobe_buffer_disable(void)908dcad1a20SNamhyung Kim static void uprobe_buffer_disable(void)
909dcad1a20SNamhyung Kim {
910dcad1a20SNamhyung Kim int cpu;
911dcad1a20SNamhyung Kim
912dcad1a20SNamhyung Kim BUG_ON(!mutex_is_locked(&event_mutex));
913dcad1a20SNamhyung Kim
914dcad1a20SNamhyung Kim if (--uprobe_buffer_refcnt == 0) {
9156ea6215fSzhangwei(Jovi) for_each_possible_cpu(cpu)
9166ea6215fSzhangwei(Jovi) free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer,
917dcad1a20SNamhyung Kim cpu)->buf);
918dcad1a20SNamhyung Kim
919dcad1a20SNamhyung Kim free_percpu(uprobe_cpu_buffer);
9206ea6215fSzhangwei(Jovi) uprobe_cpu_buffer = NULL;
9216ea6215fSzhangwei(Jovi) }
9226ea6215fSzhangwei(Jovi) }
9236ea6215fSzhangwei(Jovi)
uprobe_buffer_get(void)924dcad1a20SNamhyung Kim static struct uprobe_cpu_buffer *uprobe_buffer_get(void)
925dcad1a20SNamhyung Kim {
926dcad1a20SNamhyung Kim struct uprobe_cpu_buffer *ucb;
927dcad1a20SNamhyung Kim int cpu;
928dcad1a20SNamhyung Kim
929dcad1a20SNamhyung Kim cpu = raw_smp_processor_id();
930dcad1a20SNamhyung Kim ucb = per_cpu_ptr(uprobe_cpu_buffer, cpu);
931dcad1a20SNamhyung Kim
932dcad1a20SNamhyung Kim /*
933dcad1a20SNamhyung Kim * Use per-cpu buffers for fastest access, but we might migrate
934dcad1a20SNamhyung Kim * so the mutex makes sure we have sole access to it.
935dcad1a20SNamhyung Kim */
936dcad1a20SNamhyung Kim mutex_lock(&ucb->mutex);
937dcad1a20SNamhyung Kim
938dcad1a20SNamhyung Kim return ucb;
939dcad1a20SNamhyung Kim }
940dcad1a20SNamhyung Kim
uprobe_buffer_put(struct uprobe_cpu_buffer * ucb)941dcad1a20SNamhyung Kim static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb)
942dcad1a20SNamhyung Kim {
943dcad1a20SNamhyung Kim mutex_unlock(&ucb->mutex);
944dcad1a20SNamhyung Kim }
945dcad1a20SNamhyung Kim
__uprobe_trace_func(struct trace_uprobe * tu,unsigned long func,struct pt_regs * regs,struct uprobe_cpu_buffer * ucb,int dsize,struct trace_event_file * trace_file)946dcad1a20SNamhyung Kim static void __uprobe_trace_func(struct trace_uprobe *tu,
947dcad1a20SNamhyung Kim unsigned long func, struct pt_regs *regs,
948dcad1a20SNamhyung Kim struct uprobe_cpu_buffer *ucb, int dsize,
949dcad1a20SNamhyung Kim struct trace_event_file *trace_file)
950dcad1a20SNamhyung Kim {
951a43b9704SNamhyung Kim struct uprobe_trace_entry_head *entry;
952dd9fa555SNamhyung Kim struct trace_event_buffer fbuffer;
95370ed91c6Szhangwei(Jovi) void *data;
9547f1d2f82SSteven Rostedt (Red Hat) int size, esize;
955f3f096cfSSrikar Dronamraju struct trace_event_call *call = trace_probe_event_call(&tu->tp);
956f3f096cfSSrikar Dronamraju
957b7d5eb26SSteven Rostedt (VMware) WARN_ON(call != trace_file->event_call);
958457d1772SOleg Nesterov
959dd9fa555SNamhyung Kim if (WARN_ON_ONCE(tu->tp.size + dsize > PAGE_SIZE))
960e3dc9f89SMasami Hiramatsu return;
961f3f096cfSSrikar Dronamraju
9627f1d2f82SSteven Rostedt (Red Hat) if (trace_trigger_soft_disabled(trace_file))
96370ed91c6Szhangwei(Jovi) return;
964dd9fa555SNamhyung Kim
965a51cc604SOleg Nesterov esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
966f3f096cfSSrikar Dronamraju size = esize + tu->tp.size + dsize;
96709a5059aSSteven Rostedt (Red Hat) entry = trace_event_buffer_reserve(&fbuffer, trace_file, size);
968ca3b1620SNamhyung Kim if (!entry)
969ca3b1620SNamhyung Kim return;
970dd9fa555SNamhyung Kim
971dcad1a20SNamhyung Kim if (is_ret_probe(tu)) {
972b7d5eb26SSteven Rostedt (VMware) entry->vaddr[0] = func;
973b7d5eb26SSteven Rostedt (VMware) entry->vaddr[1] = instruction_pointer(regs);
974dd9fa555SNamhyung Kim data = DATAOF_TRACE_ENTRY(entry, true);
975dcad1a20SNamhyung Kim } else {
976393a736cSOleg Nesterov entry->vaddr[0] = instruction_pointer(regs);
977393a736cSOleg Nesterov data = DATAOF_TRACE_ENTRY(entry, false);
978393a736cSOleg Nesterov }
979393a736cSOleg Nesterov
980393a736cSOleg Nesterov memcpy(data, ucb->buf, tu->tp.size + dsize);
981457d1772SOleg Nesterov
982457d1772SOleg Nesterov trace_event_buffer_commit(&fbuffer);
983393a736cSOleg Nesterov }
984393a736cSOleg Nesterov
985dcad1a20SNamhyung Kim /* uprobe handler */
uprobe_trace_func(struct trace_uprobe * tu,struct pt_regs * regs,struct uprobe_cpu_buffer * ucb,int dsize)986f3f096cfSSrikar Dronamraju static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs,
987b7d5eb26SSteven Rostedt (VMware) struct uprobe_cpu_buffer *ucb, int dsize)
988a51cc604SOleg Nesterov {
989f42d24a1SOleg Nesterov struct event_file_link *link;
990a51cc604SOleg Nesterov
991dd9fa555SNamhyung Kim if (is_ret_probe(tu))
992dd9fa555SNamhyung Kim return 0;
993a51cc604SOleg Nesterov
99470ed91c6Szhangwei(Jovi) rcu_read_lock();
99570ed91c6Szhangwei(Jovi) trace_probe_for_each_link_rcu(link, &tu->tp)
99670ed91c6Szhangwei(Jovi) __uprobe_trace_func(tu, 0, regs, ucb, dsize, link->file);
99770ed91c6Szhangwei(Jovi) rcu_read_unlock();
99870ed91c6Szhangwei(Jovi)
99970ed91c6Szhangwei(Jovi) return 0;
1000b5f935eeSMasami Hiramatsu }
100170ed91c6Szhangwei(Jovi)
uretprobe_trace_func(struct trace_uprobe * tu,unsigned long func,struct pt_regs * regs,struct uprobe_cpu_buffer * ucb,int dsize)100270ed91c6Szhangwei(Jovi) static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func,
100370ed91c6Szhangwei(Jovi) struct pt_regs *regs,
1004f42d24a1SOleg Nesterov struct uprobe_cpu_buffer *ucb, int dsize)
1005f3f096cfSSrikar Dronamraju {
1006f3f096cfSSrikar Dronamraju struct event_file_link *link;
1007c1ae5c75SOleg Nesterov
1008dd9fa555SNamhyung Kim rcu_read_lock();
1009dd9fa555SNamhyung Kim trace_probe_for_each_link_rcu(link, &tu->tp)
1010c1ae5c75SOleg Nesterov __uprobe_trace_func(tu, func, regs, ucb, dsize, link->file);
101170ed91c6Szhangwei(Jovi) rcu_read_unlock();
101270ed91c6Szhangwei(Jovi) }
101370ed91c6Szhangwei(Jovi)
1014b5f935eeSMasami Hiramatsu /* Event entry printers */
101570ed91c6Szhangwei(Jovi) static enum print_line_t
print_uprobe_event(struct trace_iterator * iter,int flags,struct trace_event * event)101670ed91c6Szhangwei(Jovi) print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event)
1017c1ae5c75SOleg Nesterov {
1018c1ae5c75SOleg Nesterov struct uprobe_trace_entry_head *entry;
1019f3f096cfSSrikar Dronamraju struct trace_seq *s = &iter->seq;
1020f3f096cfSSrikar Dronamraju struct trace_uprobe *tu;
1021f3f096cfSSrikar Dronamraju u8 *data;
1022f3f096cfSSrikar Dronamraju
1023457d1772SOleg Nesterov entry = (struct uprobe_trace_entry_head *)iter->ent;
1024f3f096cfSSrikar Dronamraju tu = trace_uprobe_primary_from_call(
1025f3f096cfSSrikar Dronamraju container_of(event, struct trace_event_call, event));
1026f3f096cfSSrikar Dronamraju if (unlikely(!tu))
1027f3f096cfSSrikar Dronamraju goto out;
1028457d1772SOleg Nesterov
102960d53e2cSMasami Hiramatsu if (is_ret_probe(tu)) {
103060d53e2cSMasami Hiramatsu trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)",
103160d53e2cSMasami Hiramatsu trace_probe_name(&tu->tp),
103260d53e2cSMasami Hiramatsu entry->vaddr[1], entry->vaddr[0]);
1033f3f096cfSSrikar Dronamraju data = DATAOF_TRACE_ENTRY(entry, true);
10343ede82ddSOleg Nesterov } else {
10358579a107SSteven Rostedt (Red Hat) trace_seq_printf(s, "%s: (0x%lx)",
1036b55ce203SMasami Hiramatsu trace_probe_name(&tu->tp),
10378579a107SSteven Rostedt (Red Hat) entry->vaddr[0]);
10383ede82ddSOleg Nesterov data = DATAOF_TRACE_ENTRY(entry, false);
10393ede82ddSOleg Nesterov }
10408579a107SSteven Rostedt (Red Hat)
1041b55ce203SMasami Hiramatsu if (trace_probe_print_args(s, tu->tp.args, tu->tp.nr_args, data, entry) < 0)
10428579a107SSteven Rostedt (Red Hat) goto out;
1043457d1772SOleg Nesterov
10443ede82ddSOleg Nesterov trace_seq_putc(s, '\n');
10453ede82ddSOleg Nesterov
1046196b6389SSong Chen out:
10478579a107SSteven Rostedt (Red Hat) return trace_handle_return(s);
1048f3f096cfSSrikar Dronamraju }
10498579a107SSteven Rostedt (Red Hat)
1050f3f096cfSSrikar Dronamraju typedef bool (*filter_func_t)(struct uprobe_consumer *self,
10518579a107SSteven Rostedt (Red Hat) enum uprobe_filter_ctx ctx,
10528579a107SSteven Rostedt (Red Hat) struct mm_struct *mm);
1053f3f096cfSSrikar Dronamraju
trace_uprobe_enable(struct trace_uprobe * tu,filter_func_t filter)1054f3f096cfSSrikar Dronamraju static int trace_uprobe_enable(struct trace_uprobe *tu, filter_func_t filter)
105531ba3348SOleg Nesterov {
105631ba3348SOleg Nesterov int ret;
105731ba3348SOleg Nesterov
105831ba3348SOleg Nesterov tu->consumer.filter = filter;
105960d53e2cSMasami Hiramatsu tu->inode = d_real_inode(tu->path.dentry);
1060f3f096cfSSrikar Dronamraju
106170ed91c6Szhangwei(Jovi) if (tu->ref_ctr_offset)
1062f3f096cfSSrikar Dronamraju ret = uprobe_register_refctr(tu->inode, tu->offset,
106360d53e2cSMasami Hiramatsu tu->ref_ctr_offset, &tu->consumer);
106460d53e2cSMasami Hiramatsu else
106560d53e2cSMasami Hiramatsu ret = uprobe_register(tu->inode, tu->offset, &tu->consumer);
106660d53e2cSMasami Hiramatsu
106760d53e2cSMasami Hiramatsu if (ret)
106860d53e2cSMasami Hiramatsu tu->inode = NULL;
106960d53e2cSMasami Hiramatsu
107060d53e2cSMasami Hiramatsu return ret;
107160d53e2cSMasami Hiramatsu }
107260d53e2cSMasami Hiramatsu
__probe_event_disable(struct trace_probe * tp)107360d53e2cSMasami Hiramatsu static void __probe_event_disable(struct trace_probe *tp)
107460d53e2cSMasami Hiramatsu {
107560d53e2cSMasami Hiramatsu struct trace_uprobe *tu;
107660d53e2cSMasami Hiramatsu
107760d53e2cSMasami Hiramatsu tu = container_of(tp, struct trace_uprobe, tp);
107860d53e2cSMasami Hiramatsu WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
107960d53e2cSMasami Hiramatsu
108060d53e2cSMasami Hiramatsu list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
108160d53e2cSMasami Hiramatsu if (!tu->inode)
108299c9a923SMasami Hiramatsu continue;
1083b61387cbSMasami Hiramatsu
108499c9a923SMasami Hiramatsu uprobe_unregister(tu->inode, tu->offset, &tu->consumer);
1085e161c6bfSJiri Olsa tu->inode = NULL;
108660d53e2cSMasami Hiramatsu }
108760d53e2cSMasami Hiramatsu }
108860d53e2cSMasami Hiramatsu
probe_event_enable(struct trace_event_call * call,struct trace_event_file * file,filter_func_t filter)108960d53e2cSMasami Hiramatsu static int probe_event_enable(struct trace_event_call *call,
109060d53e2cSMasami Hiramatsu struct trace_event_file *file, filter_func_t filter)
109160d53e2cSMasami Hiramatsu {
109260d53e2cSMasami Hiramatsu struct trace_probe *tp;
109360d53e2cSMasami Hiramatsu struct trace_uprobe *tu;
109460d53e2cSMasami Hiramatsu bool enabled;
109560d53e2cSMasami Hiramatsu int ret;
109660d53e2cSMasami Hiramatsu
1097e161c6bfSJiri Olsa tp = trace_probe_primary_from_call(call);
109860d53e2cSMasami Hiramatsu if (WARN_ON_ONCE(!tp))
109960d53e2cSMasami Hiramatsu return -ENODEV;
110060d53e2cSMasami Hiramatsu enabled = trace_probe_is_enabled(tp);
110160d53e2cSMasami Hiramatsu
110260d53e2cSMasami Hiramatsu /* This may also change "enabled" state */
110360d53e2cSMasami Hiramatsu if (file) {
110460d53e2cSMasami Hiramatsu if (trace_probe_test_flag(tp, TP_FLAG_PROFILE))
110560d53e2cSMasami Hiramatsu return -EINTR;
110660d53e2cSMasami Hiramatsu
110760d53e2cSMasami Hiramatsu ret = trace_probe_add_file(tp, file);
110870ed91c6Szhangwei(Jovi) if (ret < 0)
110960d53e2cSMasami Hiramatsu return ret;
111048212542SOleg Nesterov } else {
111148212542SOleg Nesterov if (trace_probe_test_flag(tp, TP_FLAG_TRACE))
111260d53e2cSMasami Hiramatsu return -EINTR;
1113b5f935eeSMasami Hiramatsu
1114b5f935eeSMasami Hiramatsu trace_probe_set_flag(tp, TP_FLAG_PROFILE);
111548212542SOleg Nesterov }
111660d53e2cSMasami Hiramatsu
111748212542SOleg Nesterov tu = container_of(tp, struct trace_uprobe, tp);
111848212542SOleg Nesterov WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
111960d53e2cSMasami Hiramatsu
112048212542SOleg Nesterov if (enabled)
112170ed91c6Szhangwei(Jovi) return 0;
112260d53e2cSMasami Hiramatsu
1123b61387cbSMasami Hiramatsu ret = uprobe_buffer_enable();
1124736288baSOleg Nesterov if (ret)
112570ed91c6Szhangwei(Jovi) goto err_flags;
112670ed91c6Szhangwei(Jovi)
112770ed91c6Szhangwei(Jovi) list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1128fb6bab6aSOleg Nesterov ret = trace_uprobe_enable(tu, filter);
1129fb6bab6aSOleg Nesterov if (ret) {
1130fb6bab6aSOleg Nesterov __probe_event_disable(tp);
1131fb6bab6aSOleg Nesterov goto err_buffer;
1132e161c6bfSJiri Olsa }
113360d53e2cSMasami Hiramatsu }
113460d53e2cSMasami Hiramatsu
113560d53e2cSMasami Hiramatsu return 0;
1136fb6bab6aSOleg Nesterov
113760d53e2cSMasami Hiramatsu err_buffer:
113860d53e2cSMasami Hiramatsu uprobe_buffer_disable();
1139fb6bab6aSOleg Nesterov
1140fb6bab6aSOleg Nesterov err_flags:
1141fb6bab6aSOleg Nesterov if (file)
1142fb6bab6aSOleg Nesterov trace_probe_remove_file(tp, file);
1143fb6bab6aSOleg Nesterov else
1144fb6bab6aSOleg Nesterov trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1145fb6bab6aSOleg Nesterov
1146b5f935eeSMasami Hiramatsu return ret;
114760d53e2cSMasami Hiramatsu }
1148b5f935eeSMasami Hiramatsu
probe_event_disable(struct trace_event_call * call,struct trace_event_file * file)114960d53e2cSMasami Hiramatsu static void probe_event_disable(struct trace_event_call *call,
1150b5f935eeSMasami Hiramatsu struct trace_event_file *file)
11514161824fSOleg Nesterov {
1152f3f096cfSSrikar Dronamraju struct trace_probe *tp;
1153f3f096cfSSrikar Dronamraju
115460d53e2cSMasami Hiramatsu tp = trace_probe_primary_from_call(call);
115560d53e2cSMasami Hiramatsu if (WARN_ON_ONCE(!tp))
1156f3f096cfSSrikar Dronamraju return;
115760d53e2cSMasami Hiramatsu
115860d53e2cSMasami Hiramatsu if (!trace_probe_is_enabled(tp))
115960d53e2cSMasami Hiramatsu return;
116060d53e2cSMasami Hiramatsu
116160d53e2cSMasami Hiramatsu if (file) {
116260d53e2cSMasami Hiramatsu if (trace_probe_remove_file(tp, file) < 0)
116360d53e2cSMasami Hiramatsu return;
1164f3f096cfSSrikar Dronamraju
1165f3f096cfSSrikar Dronamraju if (trace_probe_is_enabled(tp))
116670ed91c6Szhangwei(Jovi) return;
116760d53e2cSMasami Hiramatsu } else
116870ed91c6Szhangwei(Jovi) trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
116970ed91c6Szhangwei(Jovi)
117060d53e2cSMasami Hiramatsu __probe_event_disable(tp);
117170ed91c6Szhangwei(Jovi) uprobe_buffer_disable();
1172b5f935eeSMasami Hiramatsu }
117360d53e2cSMasami Hiramatsu
uprobe_event_define_fields(struct trace_event_call * event_call)117470ed91c6Szhangwei(Jovi) static int uprobe_event_define_fields(struct trace_event_call *event_call)
117560d53e2cSMasami Hiramatsu {
1176dcad1a20SNamhyung Kim int ret, size;
1177f3f096cfSSrikar Dronamraju struct uprobe_trace_entry_head field;
1178f3f096cfSSrikar Dronamraju struct trace_uprobe *tu;
11792425bcb9SSteven Rostedt (Red Hat)
1180f3f096cfSSrikar Dronamraju tu = trace_uprobe_primary_from_call(event_call);
1181eeb07b06SMasami Hiramatsu if (unlikely(!tu))
1182f3f096cfSSrikar Dronamraju return -ENODEV;
118360d53e2cSMasami Hiramatsu
118460d53e2cSMasami Hiramatsu if (is_ret_probe(tu)) {
118560d53e2cSMasami Hiramatsu DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_FUNC, 0);
118660d53e2cSMasami Hiramatsu DEFINE_FIELD(unsigned long, vaddr[1], FIELD_STRING_RETIP, 0);
118760d53e2cSMasami Hiramatsu size = SIZEOF_TRACE_ENTRY(true);
1188f3f096cfSSrikar Dronamraju } else {
11894d1298e2SOleg Nesterov DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0);
11904d1298e2SOleg Nesterov size = SIZEOF_TRACE_ENTRY(false);
11914d1298e2SOleg Nesterov }
11924d1298e2SOleg Nesterov
11934d1298e2SOleg Nesterov return traceprobe_define_arg_fields(event_call, size, &tu->tp);
1194457d1772SOleg Nesterov }
1195457d1772SOleg Nesterov
11964d1298e2SOleg Nesterov #ifdef CONFIG_PERF_EVENTS
119714577c39SNamhyung Kim static bool
__uprobe_perf_filter(struct trace_uprobe_filter * filter,struct mm_struct * mm)1198eeb07b06SMasami Hiramatsu __uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm)
1199f3f096cfSSrikar Dronamraju {
1200f3f096cfSSrikar Dronamraju struct perf_event *event;
1201f3f096cfSSrikar Dronamraju
120231ba3348SOleg Nesterov if (filter->nr_systemwide)
120331ba3348SOleg Nesterov return true;
120431ba3348SOleg Nesterov
120531ba3348SOleg Nesterov list_for_each_entry(event, &filter->perf_events, hw.tp_list) {
120631ba3348SOleg Nesterov if (event->hw.target->mm == mm)
120731ba3348SOleg Nesterov return true;
120831ba3348SOleg Nesterov }
120931ba3348SOleg Nesterov
121031ba3348SOleg Nesterov return false;
121150f16a8bSPeter Zijlstra }
121231ba3348SOleg Nesterov
121331ba3348SOleg Nesterov static inline bool
trace_uprobe_filter_event(struct trace_uprobe_filter * filter,struct perf_event * event)121431ba3348SOleg Nesterov trace_uprobe_filter_event(struct trace_uprobe_filter *filter,
121531ba3348SOleg Nesterov struct perf_event *event)
121631ba3348SOleg Nesterov {
121731ba3348SOleg Nesterov return __uprobe_perf_filter(filter, event->hw.target->mm);
1218b2fe8ba6SOleg Nesterov }
121999c9a923SMasami Hiramatsu
trace_uprobe_filter_remove(struct trace_uprobe_filter * filter,struct perf_event * event)122099c9a923SMasami Hiramatsu static bool trace_uprobe_filter_remove(struct trace_uprobe_filter *filter,
1221b2fe8ba6SOleg Nesterov struct perf_event *event)
122299c9a923SMasami Hiramatsu {
1223b2fe8ba6SOleg Nesterov bool done;
1224b2fe8ba6SOleg Nesterov
122599c9a923SMasami Hiramatsu write_lock(&filter->rwlock);
122699c9a923SMasami Hiramatsu if (event->hw.target) {
1227ce5f36a5SOleg Nesterov list_del(&event->hw.tp_list);
1228ce5f36a5SOleg Nesterov done = filter->nr_systemwide ||
1229ce5f36a5SOleg Nesterov (event->hw.target->flags & PF_EXITING) ||
123099c9a923SMasami Hiramatsu trace_uprobe_filter_event(filter, event);
123150f16a8bSPeter Zijlstra } else {
1232ce5f36a5SOleg Nesterov filter->nr_systemwide--;
123399c9a923SMasami Hiramatsu done = filter->nr_systemwide;
123450f16a8bSPeter Zijlstra }
123599c9a923SMasami Hiramatsu write_unlock(&filter->rwlock);
1236ce5f36a5SOleg Nesterov
123799c9a923SMasami Hiramatsu return done;
123899c9a923SMasami Hiramatsu }
1239ce5f36a5SOleg Nesterov
124099c9a923SMasami Hiramatsu /* This returns true if the filter always covers target mm */
trace_uprobe_filter_add(struct trace_uprobe_filter * filter,struct perf_event * event)1241ce5f36a5SOleg Nesterov static bool trace_uprobe_filter_add(struct trace_uprobe_filter *filter,
124299c9a923SMasami Hiramatsu struct perf_event *event)
1243ce5f36a5SOleg Nesterov {
1244ce5f36a5SOleg Nesterov bool done;
124599c9a923SMasami Hiramatsu
124699c9a923SMasami Hiramatsu write_lock(&filter->rwlock);
124799c9a923SMasami Hiramatsu if (event->hw.target) {
1248736288baSOleg Nesterov /*
1249b2fe8ba6SOleg Nesterov * event->parent != NULL means copy_process(), we can avoid
1250b2fe8ba6SOleg Nesterov * uprobe_apply(). current->mm must be probed and we can rely
125199c9a923SMasami Hiramatsu * on dup_mmap() which preserves the already installed bp's.
125250f16a8bSPeter Zijlstra *
1253b2fe8ba6SOleg Nesterov * attr.enable_on_exec means that exec/mmap will install the
1254b2fe8ba6SOleg Nesterov * breakpoints we need.
1255b2fe8ba6SOleg Nesterov */
1256b2fe8ba6SOleg Nesterov done = filter->nr_systemwide ||
1257b2fe8ba6SOleg Nesterov event->parent || event->attr.enable_on_exec ||
1258b2fe8ba6SOleg Nesterov trace_uprobe_filter_event(filter, event);
1259b2fe8ba6SOleg Nesterov list_add(&event->hw.tp_list, &filter->perf_events);
1260b2fe8ba6SOleg Nesterov } else {
126199c9a923SMasami Hiramatsu done = filter->nr_systemwide;
1262b2fe8ba6SOleg Nesterov filter->nr_systemwide++;
126399c9a923SMasami Hiramatsu }
126499c9a923SMasami Hiramatsu write_unlock(&filter->rwlock);
1265b2fe8ba6SOleg Nesterov
126699c9a923SMasami Hiramatsu return done;
126799c9a923SMasami Hiramatsu }
1268b2fe8ba6SOleg Nesterov
uprobe_perf_close(struct trace_event_call * call,struct perf_event * event)126999c9a923SMasami Hiramatsu static int uprobe_perf_close(struct trace_event_call *call,
1270736288baSOleg Nesterov struct perf_event *event)
127199c9a923SMasami Hiramatsu {
1272736288baSOleg Nesterov struct trace_probe *tp;
1273736288baSOleg Nesterov struct trace_uprobe *tu;
127499c9a923SMasami Hiramatsu int ret = 0;
127599c9a923SMasami Hiramatsu
127660d53e2cSMasami Hiramatsu tp = trace_probe_primary_from_call(call);
1277e161c6bfSJiri Olsa if (WARN_ON_ONCE(!tp))
127860d53e2cSMasami Hiramatsu return -ENODEV;
127960d53e2cSMasami Hiramatsu
128060d53e2cSMasami Hiramatsu tu = container_of(tp, struct trace_uprobe, tp);
128160d53e2cSMasami Hiramatsu if (trace_uprobe_filter_remove(tu->tp.event->filter, event))
128260d53e2cSMasami Hiramatsu return 0;
128360d53e2cSMasami Hiramatsu
128460d53e2cSMasami Hiramatsu list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
128599c9a923SMasami Hiramatsu ret = uprobe_apply(tu->inode, tu->offset, &tu->consumer, false);
1286b61387cbSMasami Hiramatsu if (ret)
128799c9a923SMasami Hiramatsu break;
128899c9a923SMasami Hiramatsu }
1289e161c6bfSJiri Olsa
129099c9a923SMasami Hiramatsu return ret;
129160d53e2cSMasami Hiramatsu }
129260d53e2cSMasami Hiramatsu
uprobe_perf_open(struct trace_event_call * call,struct perf_event * event)129360d53e2cSMasami Hiramatsu static int uprobe_perf_open(struct trace_event_call *call,
129460d53e2cSMasami Hiramatsu struct perf_event *event)
129560d53e2cSMasami Hiramatsu {
129660d53e2cSMasami Hiramatsu struct trace_probe *tp;
129799c9a923SMasami Hiramatsu struct trace_uprobe *tu;
129899c9a923SMasami Hiramatsu int err = 0;
129999c9a923SMasami Hiramatsu
130099c9a923SMasami Hiramatsu tp = trace_probe_primary_from_call(call);
1301e161c6bfSJiri Olsa if (WARN_ON_ONCE(!tp))
130299c9a923SMasami Hiramatsu return -ENODEV;
130399c9a923SMasami Hiramatsu
130499c9a923SMasami Hiramatsu tu = container_of(tp, struct trace_uprobe, tp);
130599c9a923SMasami Hiramatsu if (trace_uprobe_filter_add(tu->tp.event->filter, event))
130699c9a923SMasami Hiramatsu return 0;
130799c9a923SMasami Hiramatsu
130899c9a923SMasami Hiramatsu list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
130999c9a923SMasami Hiramatsu err = uprobe_apply(tu->inode, tu->offset, &tu->consumer, true);
1310b61387cbSMasami Hiramatsu if (err) {
131199c9a923SMasami Hiramatsu uprobe_perf_close(call, event);
131299c9a923SMasami Hiramatsu break;
1313e161c6bfSJiri Olsa }
131499c9a923SMasami Hiramatsu }
131599c9a923SMasami Hiramatsu
131699c9a923SMasami Hiramatsu return err;
131799c9a923SMasami Hiramatsu }
131899c9a923SMasami Hiramatsu
uprobe_perf_filter(struct uprobe_consumer * uc,enum uprobe_filter_ctx ctx,struct mm_struct * mm)131999c9a923SMasami Hiramatsu static bool uprobe_perf_filter(struct uprobe_consumer *uc,
132099c9a923SMasami Hiramatsu enum uprobe_filter_ctx ctx, struct mm_struct *mm)
132199c9a923SMasami Hiramatsu {
132299c9a923SMasami Hiramatsu struct trace_uprobe_filter *filter;
132399c9a923SMasami Hiramatsu struct trace_uprobe *tu;
132431ba3348SOleg Nesterov int ret;
132531ba3348SOleg Nesterov
132631ba3348SOleg Nesterov tu = container_of(uc, struct trace_uprobe, consumer);
132799c9a923SMasami Hiramatsu filter = tu->tp.event->filter;
132831ba3348SOleg Nesterov
132931ba3348SOleg Nesterov read_lock(&filter->rwlock);
133031ba3348SOleg Nesterov ret = __uprobe_perf_filter(filter, mm);
133131ba3348SOleg Nesterov read_unlock(&filter->rwlock);
1332b61387cbSMasami Hiramatsu
133399c9a923SMasami Hiramatsu return ret;
133499c9a923SMasami Hiramatsu }
133599c9a923SMasami Hiramatsu
__uprobe_perf_func(struct trace_uprobe * tu,unsigned long func,struct pt_regs * regs,struct uprobe_cpu_buffer * ucb,int dsize)133699c9a923SMasami Hiramatsu static void __uprobe_perf_func(struct trace_uprobe *tu,
133731ba3348SOleg Nesterov unsigned long func, struct pt_regs *regs,
133831ba3348SOleg Nesterov struct uprobe_cpu_buffer *ucb, int dsize)
133931ba3348SOleg Nesterov {
134031ba3348SOleg Nesterov struct trace_event_call *call = trace_probe_event_call(&tu->tp);
1341a43b9704SNamhyung Kim struct uprobe_trace_entry_head *entry;
1342dd9fa555SNamhyung Kim struct hlist_head *head;
1343dd9fa555SNamhyung Kim void *data;
1344f3f096cfSSrikar Dronamraju int size, esize;
1345e3dc9f89SMasami Hiramatsu int rctx;
1346f3f096cfSSrikar Dronamraju
1347f3f096cfSSrikar Dronamraju #ifdef CONFIG_BPF_EVENTS
1348457d1772SOleg Nesterov if (bpf_prog_array_valid(call)) {
1349dd9fa555SNamhyung Kim u32 ret;
1350dcad1a20SNamhyung Kim
1351f3f096cfSSrikar Dronamraju ret = bpf_prog_run_array_uprobe(call->prog_array, regs, bpf_prog_run);
1352aca80dd9SDelyan Kratunov if (!ret)
135370ed0706SAlexei Starovoitov return;
135470ed0706SAlexei Starovoitov }
135570ed0706SAlexei Starovoitov #endif /* CONFIG_BPF_EVENTS */
13568c7dcb84SDelyan Kratunov
135770ed0706SAlexei Starovoitov esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
135804a22faeSWang Nan
135970ed0706SAlexei Starovoitov size = esize + tu->tp.size + dsize;
1360aca80dd9SDelyan Kratunov size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32);
136104a22faeSWang Nan if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough"))
1362dcad1a20SNamhyung Kim return;
1363dcad1a20SNamhyung Kim
1364dcad1a20SNamhyung Kim preempt_disable();
1365dcad1a20SNamhyung Kim head = this_cpu_ptr(call->perf_events);
1366dcad1a20SNamhyung Kim if (hlist_empty(head))
1367dcad1a20SNamhyung Kim goto out;
1368dcad1a20SNamhyung Kim
1369f3f096cfSSrikar Dronamraju entry = perf_trace_buf_alloc(size, NULL, &rctx);
1370515619f2SOleg Nesterov if (!entry)
1371515619f2SOleg Nesterov goto out;
1372515619f2SOleg Nesterov
1373515619f2SOleg Nesterov if (is_ret_probe(tu)) {
13741e1dcd93SAlexei Starovoitov entry->vaddr[0] = func;
1375f3f096cfSSrikar Dronamraju entry->vaddr[1] = instruction_pointer(regs);
1376f3f096cfSSrikar Dronamraju data = DATAOF_TRACE_ENTRY(entry, true);
1377f3f096cfSSrikar Dronamraju } else {
1378393a736cSOleg Nesterov entry->vaddr[0] = instruction_pointer(regs);
1379393a736cSOleg Nesterov data = DATAOF_TRACE_ENTRY(entry, false);
138032520b2cSOleg Nesterov }
1381393a736cSOleg Nesterov
1382393a736cSOleg Nesterov memcpy(data, ucb->buf, tu->tp.size + dsize);
138332520b2cSOleg Nesterov
1384457d1772SOleg Nesterov if (size - esize > tu->tp.size + dsize) {
1385393a736cSOleg Nesterov int len = tu->tp.size + dsize;
1386393a736cSOleg Nesterov
1387dcad1a20SNamhyung Kim memset(data + len, 0, size - esize - len);
138814577c39SNamhyung Kim }
1389dcad1a20SNamhyung Kim
1390dcad1a20SNamhyung Kim perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1391dcad1a20SNamhyung Kim head, NULL);
1392dcad1a20SNamhyung Kim out:
139314577c39SNamhyung Kim preempt_enable();
1394f3f096cfSSrikar Dronamraju }
13951e1dcd93SAlexei Starovoitov
13968fd0fbbeSPeter Zijlstra /* uprobe profile handler */
uprobe_perf_func(struct trace_uprobe * tu,struct pt_regs * regs,struct uprobe_cpu_buffer * ucb,int dsize)1397f3f096cfSSrikar Dronamraju static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs,
1398f3f096cfSSrikar Dronamraju struct uprobe_cpu_buffer *ucb, int dsize)
1399a51cc604SOleg Nesterov {
1400a51cc604SOleg Nesterov if (!uprobe_perf_filter(&tu->consumer, 0, current->mm))
1401a51cc604SOleg Nesterov return UPROBE_HANDLER_REMOVE;
1402dd9fa555SNamhyung Kim
1403dd9fa555SNamhyung Kim if (!is_ret_probe(tu))
1404a51cc604SOleg Nesterov __uprobe_perf_func(tu, 0, regs, ucb, dsize);
1405a51cc604SOleg Nesterov return 0;
1406a51cc604SOleg Nesterov }
1407a51cc604SOleg Nesterov
uretprobe_perf_func(struct trace_uprobe * tu,unsigned long func,struct pt_regs * regs,struct uprobe_cpu_buffer * ucb,int dsize)1408393a736cSOleg Nesterov static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func,
1409dd9fa555SNamhyung Kim struct pt_regs *regs,
1410f42d24a1SOleg Nesterov struct uprobe_cpu_buffer *ucb, int dsize)
1411f3f096cfSSrikar Dronamraju {
1412c1ae5c75SOleg Nesterov __uprobe_perf_func(tu, func, regs, ucb, dsize);
1413c1ae5c75SOleg Nesterov }
1414dd9fa555SNamhyung Kim
bpf_get_uprobe_info(const struct perf_event * event,u32 * fd_type,const char ** filename,u64 * probe_offset,u64 * probe_addr,bool perf_type_tracepoint)1415dd9fa555SNamhyung Kim int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type,
1416c1ae5c75SOleg Nesterov const char **filename, u64 *probe_offset,
1417dd9fa555SNamhyung Kim u64 *probe_addr, bool perf_type_tracepoint)
1418c1ae5c75SOleg Nesterov {
141941bdc4b4SYonghong Song const char *pevent = trace_event_name(event->tp_event);
142041bdc4b4SYonghong Song const char *group = event->tp_event->class->system;
142141bdc4b4SYonghong Song struct trace_uprobe *tu;
142241bdc4b4SYonghong Song
142341bdc4b4SYonghong Song if (perf_type_tracepoint)
142441bdc4b4SYonghong Song tu = find_probe_event(pevent, group);
142541bdc4b4SYonghong Song else
142641bdc4b4SYonghong Song tu = trace_uprobe_primary_from_call(event->tp_event);
142741bdc4b4SYonghong Song if (!tu)
142841bdc4b4SYonghong Song return -EINVAL;
142941bdc4b4SYonghong Song
143041bdc4b4SYonghong Song *fd_type = is_ret_probe(tu) ? BPF_FD_TYPE_URETPROBE
143122d5bd68SJean-Philippe Brucker : BPF_FD_TYPE_UPROBE;
143241bdc4b4SYonghong Song *filename = tu->filename;
143341bdc4b4SYonghong Song *probe_offset = tu->offset;
143441bdc4b4SYonghong Song *probe_addr = 0;
143541bdc4b4SYonghong Song return 0;
143641bdc4b4SYonghong Song }
143741bdc4b4SYonghong Song #endif /* CONFIG_PERF_EVENTS */
143841bdc4b4SYonghong Song
143941bdc4b4SYonghong Song static int
trace_uprobe_register(struct trace_event_call * event,enum trace_reg type,void * data)144041bdc4b4SYonghong Song trace_uprobe_register(struct trace_event_call *event, enum trace_reg type,
1441f3f096cfSSrikar Dronamraju void *data)
1442f3f096cfSSrikar Dronamraju {
144370ed91c6Szhangwei(Jovi) struct trace_event_file *file = data;
14442425bcb9SSteven Rostedt (Red Hat)
144570ed91c6Szhangwei(Jovi) switch (type) {
1446f3f096cfSSrikar Dronamraju case TRACE_REG_REGISTER:
14477f1d2f82SSteven Rostedt (Red Hat) return probe_event_enable(event, file, NULL);
1448f3f096cfSSrikar Dronamraju
1449f3f096cfSSrikar Dronamraju case TRACE_REG_UNREGISTER:
1450f3f096cfSSrikar Dronamraju probe_event_disable(event, file);
145160d53e2cSMasami Hiramatsu return 0;
1452f3f096cfSSrikar Dronamraju
1453f3f096cfSSrikar Dronamraju #ifdef CONFIG_PERF_EVENTS
145460d53e2cSMasami Hiramatsu case TRACE_REG_PERF_REGISTER:
1455f3f096cfSSrikar Dronamraju return probe_event_enable(event, NULL, uprobe_perf_filter);
1456f3f096cfSSrikar Dronamraju
1457f3f096cfSSrikar Dronamraju case TRACE_REG_PERF_UNREGISTER:
1458f3f096cfSSrikar Dronamraju probe_event_disable(event, NULL);
145960d53e2cSMasami Hiramatsu return 0;
1460f3f096cfSSrikar Dronamraju
1461f3f096cfSSrikar Dronamraju case TRACE_REG_PERF_OPEN:
146260d53e2cSMasami Hiramatsu return uprobe_perf_open(event, data);
1463f3f096cfSSrikar Dronamraju
1464736288baSOleg Nesterov case TRACE_REG_PERF_CLOSE:
1465736288baSOleg Nesterov return uprobe_perf_close(event, data);
146699c9a923SMasami Hiramatsu
1467736288baSOleg Nesterov #endif
1468736288baSOleg Nesterov default:
146999c9a923SMasami Hiramatsu return 0;
1470736288baSOleg Nesterov }
1471f3f096cfSSrikar Dronamraju }
1472f3f096cfSSrikar Dronamraju
uprobe_dispatcher(struct uprobe_consumer * con,struct pt_regs * regs)1473f3f096cfSSrikar Dronamraju static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)
1474f3f096cfSSrikar Dronamraju {
1475f3f096cfSSrikar Dronamraju struct trace_uprobe *tu;
1476f3f096cfSSrikar Dronamraju struct uprobe_dispatch_data udd;
1477f3f096cfSSrikar Dronamraju struct uprobe_cpu_buffer *ucb;
1478f3f096cfSSrikar Dronamraju int dsize, esize;
1479f3f096cfSSrikar Dronamraju int ret = 0;
1480b7e0bf34SNamhyung Kim
1481dd9fa555SNamhyung Kim
1482dd9fa555SNamhyung Kim tu = container_of(con, struct trace_uprobe, consumer);
1483f42d24a1SOleg Nesterov tu->nhit++;
1484f3f096cfSSrikar Dronamraju
1485dd9fa555SNamhyung Kim udd.tu = tu;
1486a932b738SOleg Nesterov udd.bp_addr = instruction_pointer(regs);
14871b47aefdSOleg Nesterov
1488f3f096cfSSrikar Dronamraju current->utask->vaddr = (unsigned long) &udd;
1489b7e0bf34SNamhyung Kim
1490b7e0bf34SNamhyung Kim if (WARN_ON_ONCE(!uprobe_cpu_buffer))
1491b7e0bf34SNamhyung Kim return 0;
1492b7e0bf34SNamhyung Kim
1493b7e0bf34SNamhyung Kim dsize = __get_data_size(&tu->tp, regs);
1494dd9fa555SNamhyung Kim esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1495dd9fa555SNamhyung Kim
1496dd9fa555SNamhyung Kim ucb = uprobe_buffer_get();
1497dd9fa555SNamhyung Kim store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
1498dd9fa555SNamhyung Kim
1499dd9fa555SNamhyung Kim if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))
1500dd9fa555SNamhyung Kim ret |= uprobe_trace_func(tu, regs, ucb, dsize);
15019178412dSMasami Hiramatsu
1502dd9fa555SNamhyung Kim #ifdef CONFIG_PERF_EVENTS
1503747774d6SMasami Hiramatsu if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))
1504dd9fa555SNamhyung Kim ret |= uprobe_perf_func(tu, regs, ucb, dsize);
1505f3f096cfSSrikar Dronamraju #endif
1506f3f096cfSSrikar Dronamraju uprobe_buffer_put(ucb);
1507747774d6SMasami Hiramatsu return ret;
1508dd9fa555SNamhyung Kim }
1509f3f096cfSSrikar Dronamraju
uretprobe_dispatcher(struct uprobe_consumer * con,unsigned long func,struct pt_regs * regs)1510dd9fa555SNamhyung Kim static int uretprobe_dispatcher(struct uprobe_consumer *con,
1511f42d24a1SOleg Nesterov unsigned long func, struct pt_regs *regs)
1512f3f096cfSSrikar Dronamraju {
1513f3f096cfSSrikar Dronamraju struct trace_uprobe *tu;
1514c1ae5c75SOleg Nesterov struct uprobe_dispatch_data udd;
1515c1ae5c75SOleg Nesterov struct uprobe_cpu_buffer *ucb;
1516c1ae5c75SOleg Nesterov int dsize, esize;
1517c1ae5c75SOleg Nesterov
1518b7e0bf34SNamhyung Kim tu = container_of(con, struct trace_uprobe, consumer);
1519dd9fa555SNamhyung Kim
1520dd9fa555SNamhyung Kim udd.tu = tu;
1521c1ae5c75SOleg Nesterov udd.bp_addr = func;
1522c1ae5c75SOleg Nesterov
1523c1ae5c75SOleg Nesterov current->utask->vaddr = (unsigned long) &udd;
1524b7e0bf34SNamhyung Kim
1525b7e0bf34SNamhyung Kim if (WARN_ON_ONCE(!uprobe_cpu_buffer))
1526b7e0bf34SNamhyung Kim return 0;
1527b7e0bf34SNamhyung Kim
1528b7e0bf34SNamhyung Kim dsize = __get_data_size(&tu->tp, regs);
1529dd9fa555SNamhyung Kim esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1530dd9fa555SNamhyung Kim
1531dd9fa555SNamhyung Kim ucb = uprobe_buffer_get();
1532dd9fa555SNamhyung Kim store_trace_args(ucb->buf, &tu->tp, regs, esize, dsize);
1533dd9fa555SNamhyung Kim
1534dd9fa555SNamhyung Kim if (trace_probe_test_flag(&tu->tp, TP_FLAG_TRACE))
1535dd9fa555SNamhyung Kim uretprobe_trace_func(tu, func, regs, ucb, dsize);
15369178412dSMasami Hiramatsu
1537dd9fa555SNamhyung Kim #ifdef CONFIG_PERF_EVENTS
1538747774d6SMasami Hiramatsu if (trace_probe_test_flag(&tu->tp, TP_FLAG_PROFILE))
1539dd9fa555SNamhyung Kim uretprobe_perf_func(tu, func, regs, ucb, dsize);
1540c1ae5c75SOleg Nesterov #endif
1541c1ae5c75SOleg Nesterov uprobe_buffer_put(ucb);
1542747774d6SMasami Hiramatsu return 0;
1543dd9fa555SNamhyung Kim }
1544c1ae5c75SOleg Nesterov
1545dd9fa555SNamhyung Kim static struct trace_event_functions uprobe_funcs = {
1546c1ae5c75SOleg Nesterov .trace = print_uprobe_event
1547c1ae5c75SOleg Nesterov };
1548c1ae5c75SOleg Nesterov
1549f3f096cfSSrikar Dronamraju static struct trace_event_fields uprobe_fields_array[] = {
1550f3f096cfSSrikar Dronamraju { .type = TRACE_FUNCTION_TYPE,
1551f3f096cfSSrikar Dronamraju .define_fields = uprobe_event_define_fields },
1552f3f096cfSSrikar Dronamraju {}
155304ae87a5SPeter Zijlstra };
155404ae87a5SPeter Zijlstra
init_trace_event_call(struct trace_uprobe * tu)155504ae87a5SPeter Zijlstra static inline void init_trace_event_call(struct trace_uprobe *tu)
155604ae87a5SPeter Zijlstra {
155704ae87a5SPeter Zijlstra struct trace_event_call *call = trace_probe_event_call(&tu->tp);
155804ae87a5SPeter Zijlstra call->event.funcs = &uprobe_funcs;
1559e3dc9f89SMasami Hiramatsu call->class->fields_array = uprobe_fields_array;
1560f3f096cfSSrikar Dronamraju
1561e3dc9f89SMasami Hiramatsu call->flags = TRACE_EVENT_FL_UPROBE | TRACE_EVENT_FL_CAP_ANY;
1562f3f096cfSSrikar Dronamraju call->class->reg = trace_uprobe_register;
156304ae87a5SPeter Zijlstra }
1564f3f096cfSSrikar Dronamraju
register_uprobe_event(struct trace_uprobe * tu)15659fd2e48bSSong Liu static int register_uprobe_event(struct trace_uprobe *tu)
156633ea4b24SSong Liu {
156733ea4b24SSong Liu init_trace_event_call(tu);
156833ea4b24SSong Liu
156933ea4b24SSong Liu return trace_probe_register_event_call(&tu->tp);
157033ea4b24SSong Liu }
1571e3dc9f89SMasami Hiramatsu
unregister_uprobe_event(struct trace_uprobe * tu)157233ea4b24SSong Liu static int unregister_uprobe_event(struct trace_uprobe *tu)
157346e5376dSMasami Hiramatsu {
1574f3f096cfSSrikar Dronamraju return trace_probe_unregister_event_call(&tu->tp);
1575f3f096cfSSrikar Dronamraju }
1576c6c2401dSSteven Rostedt (Red Hat)
1577f3f096cfSSrikar Dronamraju #ifdef CONFIG_PERF_EVENTS
157846e5376dSMasami Hiramatsu struct trace_event_call *
create_local_trace_uprobe(char * name,unsigned long offs,unsigned long ref_ctr_offset,bool is_return)1579f3f096cfSSrikar Dronamraju create_local_trace_uprobe(char *name, unsigned long offs,
1580f3f096cfSSrikar Dronamraju unsigned long ref_ctr_offset, bool is_return)
158133ea4b24SSong Liu {
158233ea4b24SSong Liu enum probe_print_type ptype;
1583a6ca88b2SSong Liu struct trace_uprobe *tu;
1584a6ca88b2SSong Liu struct path path;
158533ea4b24SSong Liu int ret;
1586007517a0SSteven Rostedt (VMware)
158733ea4b24SSong Liu ret = kern_path(name, LOOKUP_FOLLOW, &path);
158833ea4b24SSong Liu if (ret)
158933ea4b24SSong Liu return ERR_PTR(ret);
159033ea4b24SSong Liu
159133ea4b24SSong Liu if (!d_is_reg(path.dentry)) {
159233ea4b24SSong Liu path_put(&path);
159333ea4b24SSong Liu return ERR_PTR(-EINVAL);
159433ea4b24SSong Liu }
15950c92c7a3SSong Liu
159633ea4b24SSong Liu /*
159733ea4b24SSong Liu * local trace_kprobes are not added to dyn_event, so they are never
159833ea4b24SSong Liu * searched in find_trace_kprobe(). Therefore, there is no concern of
159933ea4b24SSong Liu * duplicated name "DUMMY_EVENT" here.
160033ea4b24SSong Liu */
16010597c49cSMasami Hiramatsu tu = alloc_trace_uprobe(UPROBE_EVENT_SYSTEM, "DUMMY_EVENT", 0,
160233ea4b24SSong Liu is_return);
160333ea4b24SSong Liu
160433ea4b24SSong Liu if (IS_ERR(tu)) {
160533ea4b24SSong Liu pr_info("Failed to allocate trace_uprobe.(%d)\n",
160633ea4b24SSong Liu (int)PTR_ERR(tu));
160733ea4b24SSong Liu path_put(&path);
160833ea4b24SSong Liu return ERR_CAST(tu);
160933ea4b24SSong Liu }
161033ea4b24SSong Liu
16110c92c7a3SSong Liu tu->offset = offs;
161233ea4b24SSong Liu tu->path = path;
161333ea4b24SSong Liu tu->ref_ctr_offset = ref_ctr_offset;
161433ea4b24SSong Liu tu->filename = kstrdup(name, GFP_KERNEL);
161533ea4b24SSong Liu if (!tu->filename) {
16160c92c7a3SSong Liu ret = -ENOMEM;
1617a6ca88b2SSong Liu goto error;
161833ea4b24SSong Liu }
16198c722424SXiaoke Wang
16208c722424SXiaoke Wang init_trace_event_call(tu);
16218c722424SXiaoke Wang
16228c722424SXiaoke Wang ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
16238c722424SXiaoke Wang if (traceprobe_set_print_fmt(&tu->tp, ptype) < 0) {
1624e3dc9f89SMasami Hiramatsu ret = -ENOMEM;
162533ea4b24SSong Liu goto error;
1626007517a0SSteven Rostedt (VMware) }
1627007517a0SSteven Rostedt (VMware)
162833ea4b24SSong Liu return trace_probe_event_call(&tu->tp);
162933ea4b24SSong Liu error:
163033ea4b24SSong Liu free_trace_uprobe(tu);
163133ea4b24SSong Liu return ERR_PTR(ret);
1632e3dc9f89SMasami Hiramatsu }
163333ea4b24SSong Liu
destroy_local_trace_uprobe(struct trace_event_call * event_call)163433ea4b24SSong Liu void destroy_local_trace_uprobe(struct trace_event_call *event_call)
163533ea4b24SSong Liu {
163633ea4b24SSong Liu struct trace_uprobe *tu;
163733ea4b24SSong Liu
163833ea4b24SSong Liu tu = trace_uprobe_primary_from_call(event_call);
163933ea4b24SSong Liu
164033ea4b24SSong Liu free_trace_uprobe(tu);
164133ea4b24SSong Liu }
164260d53e2cSMasami Hiramatsu #endif /* CONFIG_PERF_EVENTS */
164333ea4b24SSong Liu
164433ea4b24SSong Liu /* Make a trace interface for controlling probe points */
init_uprobe_trace(void)164533ea4b24SSong Liu static __init int init_uprobe_trace(void)
164633ea4b24SSong Liu {
164733ea4b24SSong Liu int ret;
164839bcdd6aSBhaskar Chowdhury
1649f3f096cfSSrikar Dronamraju ret = dyn_event_register(&trace_uprobe_ops);
1650f3f096cfSSrikar Dronamraju if (ret)
16510597c49cSMasami Hiramatsu return ret;
16520597c49cSMasami Hiramatsu
16530597c49cSMasami Hiramatsu ret = tracing_init_dentry();
16540597c49cSMasami Hiramatsu if (ret)
16550597c49cSMasami Hiramatsu return 0;
1656f3f096cfSSrikar Dronamraju
165722c36b18SWei Yang trace_create_file("uprobe_events", TRACE_MODE_WRITE, NULL,
165822c36b18SWei Yang NULL, &uprobe_events_ops);
1659f3f096cfSSrikar Dronamraju /* Profile interface */
1660f3f096cfSSrikar Dronamraju trace_create_file("uprobe_profile", TRACE_MODE_READ, NULL,
166121ccc9cdSSteven Rostedt (VMware) NULL, &uprobe_profile_ops);
1662f3f096cfSSrikar Dronamraju return 0;
1663f3f096cfSSrikar Dronamraju }
166421ccc9cdSSteven Rostedt (VMware)
1665f3f096cfSSrikar Dronamraju fs_initcall(init_uprobe_trace);
1666f3f096cfSSrikar Dronamraju