1179a0cc4SSteven Rostedt (VMware) // SPDX-License-Identifier: GPL-2.0 22541517cSAlexei Starovoitov /* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com 30515e599SAlexei Starovoitov * Copyright (c) 2016 Facebook 42541517cSAlexei Starovoitov */ 52541517cSAlexei Starovoitov #include <linux/kernel.h> 62541517cSAlexei Starovoitov #include <linux/types.h> 72541517cSAlexei Starovoitov #include <linux/slab.h> 82541517cSAlexei Starovoitov #include <linux/bpf.h> 90515e599SAlexei Starovoitov #include <linux/bpf_perf_event.h> 102541517cSAlexei Starovoitov #include <linux/filter.h> 112541517cSAlexei Starovoitov #include <linux/uaccess.h> 129c959c86SAlexei Starovoitov #include <linux/ctype.h> 139802d865SJosef Bacik #include <linux/kprobes.h> 1441bdc4b4SYonghong Song #include <linux/syscalls.h> 15540adea3SMasami Hiramatsu #include <linux/error-injection.h> 169802d865SJosef Bacik 17c7b6f29bSNadav Amit #include <asm/tlb.h> 18c7b6f29bSNadav Amit 199802d865SJosef Bacik #include "trace_probe.h" 202541517cSAlexei Starovoitov #include "trace.h" 212541517cSAlexei Starovoitov 22e672db03SStanislav Fomichev #define bpf_event_rcu_dereference(p) \ 23e672db03SStanislav Fomichev rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex)) 24e672db03SStanislav Fomichev 25a38d1107SMatt Mullins #ifdef CONFIG_MODULES 26a38d1107SMatt Mullins struct bpf_trace_module { 27a38d1107SMatt Mullins struct module *module; 28a38d1107SMatt Mullins struct list_head list; 29a38d1107SMatt Mullins }; 30a38d1107SMatt Mullins 31a38d1107SMatt Mullins static LIST_HEAD(bpf_trace_modules); 32a38d1107SMatt Mullins static DEFINE_MUTEX(bpf_module_mutex); 33a38d1107SMatt Mullins 34a38d1107SMatt Mullins static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name) 35a38d1107SMatt Mullins { 36a38d1107SMatt Mullins struct bpf_raw_event_map *btp, *ret = NULL; 37a38d1107SMatt Mullins struct bpf_trace_module *btm; 38a38d1107SMatt Mullins unsigned int i; 39a38d1107SMatt Mullins 40a38d1107SMatt Mullins mutex_lock(&bpf_module_mutex); 41a38d1107SMatt Mullins list_for_each_entry(btm, &bpf_trace_modules, list) { 42a38d1107SMatt Mullins for (i = 0; i < btm->module->num_bpf_raw_events; ++i) { 43a38d1107SMatt Mullins btp = &btm->module->bpf_raw_events[i]; 44a38d1107SMatt Mullins if (!strcmp(btp->tp->name, name)) { 45a38d1107SMatt Mullins if (try_module_get(btm->module)) 46a38d1107SMatt Mullins ret = btp; 47a38d1107SMatt Mullins goto out; 48a38d1107SMatt Mullins } 49a38d1107SMatt Mullins } 50a38d1107SMatt Mullins } 51a38d1107SMatt Mullins out: 52a38d1107SMatt Mullins mutex_unlock(&bpf_module_mutex); 53a38d1107SMatt Mullins return ret; 54a38d1107SMatt Mullins } 55a38d1107SMatt Mullins #else 56a38d1107SMatt Mullins static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name) 57a38d1107SMatt Mullins { 58a38d1107SMatt Mullins return NULL; 59a38d1107SMatt Mullins } 60a38d1107SMatt Mullins #endif /* CONFIG_MODULES */ 61a38d1107SMatt Mullins 62035226b9SGianluca Borello u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 63c195651eSYonghong Song u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 64035226b9SGianluca Borello 652541517cSAlexei Starovoitov /** 662541517cSAlexei Starovoitov * trace_call_bpf - invoke BPF program 67e87c6bc3SYonghong Song * @call: tracepoint event 682541517cSAlexei Starovoitov * @ctx: opaque context pointer 692541517cSAlexei Starovoitov * 702541517cSAlexei Starovoitov * kprobe handlers execute BPF programs via this helper. 712541517cSAlexei Starovoitov * Can be used from static tracepoints in the future. 722541517cSAlexei Starovoitov * 732541517cSAlexei Starovoitov * Return: BPF programs always return an integer which is interpreted by 742541517cSAlexei Starovoitov * kprobe handler as: 752541517cSAlexei Starovoitov * 0 - return from kprobe (event is filtered out) 762541517cSAlexei Starovoitov * 1 - store kprobe event into ring buffer 772541517cSAlexei Starovoitov * Other values are reserved and currently alias to 1 782541517cSAlexei Starovoitov */ 79e87c6bc3SYonghong Song unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) 802541517cSAlexei Starovoitov { 812541517cSAlexei Starovoitov unsigned int ret; 822541517cSAlexei Starovoitov 832541517cSAlexei Starovoitov if (in_nmi()) /* not supported yet */ 842541517cSAlexei Starovoitov return 1; 852541517cSAlexei Starovoitov 862541517cSAlexei Starovoitov preempt_disable(); 872541517cSAlexei Starovoitov 882541517cSAlexei Starovoitov if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { 892541517cSAlexei Starovoitov /* 902541517cSAlexei Starovoitov * since some bpf program is already running on this cpu, 912541517cSAlexei Starovoitov * don't call into another bpf program (same or different) 922541517cSAlexei Starovoitov * and don't send kprobe event into ring-buffer, 932541517cSAlexei Starovoitov * so return zero here 942541517cSAlexei Starovoitov */ 952541517cSAlexei Starovoitov ret = 0; 962541517cSAlexei Starovoitov goto out; 972541517cSAlexei Starovoitov } 982541517cSAlexei Starovoitov 99e87c6bc3SYonghong Song /* 100e87c6bc3SYonghong Song * Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock 101e87c6bc3SYonghong Song * to all call sites, we did a bpf_prog_array_valid() there to check 102e87c6bc3SYonghong Song * whether call->prog_array is empty or not, which is 103e87c6bc3SYonghong Song * a heurisitc to speed up execution. 104e87c6bc3SYonghong Song * 105e87c6bc3SYonghong Song * If bpf_prog_array_valid() fetched prog_array was 106e87c6bc3SYonghong Song * non-NULL, we go into trace_call_bpf() and do the actual 107e87c6bc3SYonghong Song * proper rcu_dereference() under RCU lock. 108e87c6bc3SYonghong Song * If it turns out that prog_array is NULL then, we bail out. 109e87c6bc3SYonghong Song * For the opposite, if the bpf_prog_array_valid() fetched pointer 110e87c6bc3SYonghong Song * was NULL, you'll skip the prog_array with the risk of missing 111e87c6bc3SYonghong Song * out of events when it was updated in between this and the 112e87c6bc3SYonghong Song * rcu_dereference() which is accepted risk. 113e87c6bc3SYonghong Song */ 114e87c6bc3SYonghong Song ret = BPF_PROG_RUN_ARRAY_CHECK(call->prog_array, ctx, BPF_PROG_RUN); 1152541517cSAlexei Starovoitov 1162541517cSAlexei Starovoitov out: 1172541517cSAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 1182541517cSAlexei Starovoitov preempt_enable(); 1192541517cSAlexei Starovoitov 1202541517cSAlexei Starovoitov return ret; 1212541517cSAlexei Starovoitov } 1222541517cSAlexei Starovoitov EXPORT_SYMBOL_GPL(trace_call_bpf); 1232541517cSAlexei Starovoitov 1249802d865SJosef Bacik #ifdef CONFIG_BPF_KPROBE_OVERRIDE 1259802d865SJosef Bacik BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc) 1269802d865SJosef Bacik { 1279802d865SJosef Bacik regs_set_return_value(regs, rc); 128540adea3SMasami Hiramatsu override_function_with_return(regs); 1299802d865SJosef Bacik return 0; 1309802d865SJosef Bacik } 1319802d865SJosef Bacik 1329802d865SJosef Bacik static const struct bpf_func_proto bpf_override_return_proto = { 1339802d865SJosef Bacik .func = bpf_override_return, 1349802d865SJosef Bacik .gpl_only = true, 1359802d865SJosef Bacik .ret_type = RET_INTEGER, 1369802d865SJosef Bacik .arg1_type = ARG_PTR_TO_CTX, 1379802d865SJosef Bacik .arg2_type = ARG_ANYTHING, 1389802d865SJosef Bacik }; 1399802d865SJosef Bacik #endif 1409802d865SJosef Bacik 1416ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_user, void *, dst, u32, size, 1426ae08ae3SDaniel Borkmann const void __user *, unsafe_ptr) 1432541517cSAlexei Starovoitov { 1446ae08ae3SDaniel Borkmann int ret = probe_user_read(dst, unsafe_ptr, size); 1452541517cSAlexei Starovoitov 146074f528eSDaniel Borkmann if (unlikely(ret < 0)) 147074f528eSDaniel Borkmann memset(dst, 0, size); 148074f528eSDaniel Borkmann 149074f528eSDaniel Borkmann return ret; 1502541517cSAlexei Starovoitov } 1512541517cSAlexei Starovoitov 1526ae08ae3SDaniel Borkmann static const struct bpf_func_proto bpf_probe_read_user_proto = { 1536ae08ae3SDaniel Borkmann .func = bpf_probe_read_user, 1546ae08ae3SDaniel Borkmann .gpl_only = true, 1556ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 1566ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 1576ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 1586ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 1596ae08ae3SDaniel Borkmann }; 1606ae08ae3SDaniel Borkmann 1616ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_user_str, void *, dst, u32, size, 1626ae08ae3SDaniel Borkmann const void __user *, unsafe_ptr) 1636ae08ae3SDaniel Borkmann { 1646ae08ae3SDaniel Borkmann int ret = strncpy_from_unsafe_user(dst, unsafe_ptr, size); 1656ae08ae3SDaniel Borkmann 1666ae08ae3SDaniel Borkmann if (unlikely(ret < 0)) 1676ae08ae3SDaniel Borkmann memset(dst, 0, size); 1686ae08ae3SDaniel Borkmann 1696ae08ae3SDaniel Borkmann return ret; 1706ae08ae3SDaniel Borkmann } 1716ae08ae3SDaniel Borkmann 1726ae08ae3SDaniel Borkmann static const struct bpf_func_proto bpf_probe_read_user_str_proto = { 1736ae08ae3SDaniel Borkmann .func = bpf_probe_read_user_str, 1746ae08ae3SDaniel Borkmann .gpl_only = true, 1756ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 1766ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 1776ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 1786ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 1796ae08ae3SDaniel Borkmann }; 1806ae08ae3SDaniel Borkmann 1816ae08ae3SDaniel Borkmann static __always_inline int 1826ae08ae3SDaniel Borkmann bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr, 1836ae08ae3SDaniel Borkmann const bool compat) 1846ae08ae3SDaniel Borkmann { 1856ae08ae3SDaniel Borkmann int ret = security_locked_down(LOCKDOWN_BPF_READ); 1866ae08ae3SDaniel Borkmann 1876ae08ae3SDaniel Borkmann if (unlikely(ret < 0)) 1886ae08ae3SDaniel Borkmann goto out; 1896ae08ae3SDaniel Borkmann ret = compat ? probe_kernel_read(dst, unsafe_ptr, size) : 1906ae08ae3SDaniel Borkmann probe_kernel_read_strict(dst, unsafe_ptr, size); 1916ae08ae3SDaniel Borkmann if (unlikely(ret < 0)) 1926ae08ae3SDaniel Borkmann out: 1936ae08ae3SDaniel Borkmann memset(dst, 0, size); 1946ae08ae3SDaniel Borkmann return ret; 1956ae08ae3SDaniel Borkmann } 1966ae08ae3SDaniel Borkmann 1976ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_kernel, void *, dst, u32, size, 1986ae08ae3SDaniel Borkmann const void *, unsafe_ptr) 1996ae08ae3SDaniel Borkmann { 2006ae08ae3SDaniel Borkmann return bpf_probe_read_kernel_common(dst, size, unsafe_ptr, false); 2016ae08ae3SDaniel Borkmann } 2026ae08ae3SDaniel Borkmann 2036ae08ae3SDaniel Borkmann static const struct bpf_func_proto bpf_probe_read_kernel_proto = { 2046ae08ae3SDaniel Borkmann .func = bpf_probe_read_kernel, 2056ae08ae3SDaniel Borkmann .gpl_only = true, 2066ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 2076ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 2086ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 2096ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 2106ae08ae3SDaniel Borkmann }; 2116ae08ae3SDaniel Borkmann 2126ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_compat, void *, dst, u32, size, 2136ae08ae3SDaniel Borkmann const void *, unsafe_ptr) 2146ae08ae3SDaniel Borkmann { 2156ae08ae3SDaniel Borkmann return bpf_probe_read_kernel_common(dst, size, unsafe_ptr, true); 2166ae08ae3SDaniel Borkmann } 2176ae08ae3SDaniel Borkmann 2186ae08ae3SDaniel Borkmann static const struct bpf_func_proto bpf_probe_read_compat_proto = { 2196ae08ae3SDaniel Borkmann .func = bpf_probe_read_compat, 2206ae08ae3SDaniel Borkmann .gpl_only = true, 2216ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 2226ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 2236ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 2246ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 2256ae08ae3SDaniel Borkmann }; 2266ae08ae3SDaniel Borkmann 2276ae08ae3SDaniel Borkmann static __always_inline int 2286ae08ae3SDaniel Borkmann bpf_probe_read_kernel_str_common(void *dst, u32 size, const void *unsafe_ptr, 2296ae08ae3SDaniel Borkmann const bool compat) 2306ae08ae3SDaniel Borkmann { 2316ae08ae3SDaniel Borkmann int ret = security_locked_down(LOCKDOWN_BPF_READ); 2326ae08ae3SDaniel Borkmann 2336ae08ae3SDaniel Borkmann if (unlikely(ret < 0)) 2346ae08ae3SDaniel Borkmann goto out; 2356ae08ae3SDaniel Borkmann /* 2366ae08ae3SDaniel Borkmann * The strncpy_from_unsafe_*() call will likely not fill the entire 2376ae08ae3SDaniel Borkmann * buffer, but that's okay in this circumstance as we're probing 2386ae08ae3SDaniel Borkmann * arbitrary memory anyway similar to bpf_probe_read_*() and might 2396ae08ae3SDaniel Borkmann * as well probe the stack. Thus, memory is explicitly cleared 2406ae08ae3SDaniel Borkmann * only in error case, so that improper users ignoring return 2416ae08ae3SDaniel Borkmann * code altogether don't copy garbage; otherwise length of string 2426ae08ae3SDaniel Borkmann * is returned that can be used for bpf_perf_event_output() et al. 2436ae08ae3SDaniel Borkmann */ 2446ae08ae3SDaniel Borkmann ret = compat ? strncpy_from_unsafe(dst, unsafe_ptr, size) : 2456ae08ae3SDaniel Borkmann strncpy_from_unsafe_strict(dst, unsafe_ptr, size); 2466ae08ae3SDaniel Borkmann if (unlikely(ret < 0)) 2476ae08ae3SDaniel Borkmann out: 2486ae08ae3SDaniel Borkmann memset(dst, 0, size); 2496ae08ae3SDaniel Borkmann return ret; 2506ae08ae3SDaniel Borkmann } 2516ae08ae3SDaniel Borkmann 2526ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_kernel_str, void *, dst, u32, size, 2536ae08ae3SDaniel Borkmann const void *, unsafe_ptr) 2546ae08ae3SDaniel Borkmann { 2556ae08ae3SDaniel Borkmann return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr, false); 2566ae08ae3SDaniel Borkmann } 2576ae08ae3SDaniel Borkmann 2586ae08ae3SDaniel Borkmann static const struct bpf_func_proto bpf_probe_read_kernel_str_proto = { 2596ae08ae3SDaniel Borkmann .func = bpf_probe_read_kernel_str, 2606ae08ae3SDaniel Borkmann .gpl_only = true, 2616ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 2626ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 2636ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 2646ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 2656ae08ae3SDaniel Borkmann }; 2666ae08ae3SDaniel Borkmann 2676ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_compat_str, void *, dst, u32, size, 2686ae08ae3SDaniel Borkmann const void *, unsafe_ptr) 2696ae08ae3SDaniel Borkmann { 2706ae08ae3SDaniel Borkmann return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr, true); 2716ae08ae3SDaniel Borkmann } 2726ae08ae3SDaniel Borkmann 2736ae08ae3SDaniel Borkmann static const struct bpf_func_proto bpf_probe_read_compat_str_proto = { 2746ae08ae3SDaniel Borkmann .func = bpf_probe_read_compat_str, 2752541517cSAlexei Starovoitov .gpl_only = true, 2762541517cSAlexei Starovoitov .ret_type = RET_INTEGER, 27739f19ebbSAlexei Starovoitov .arg1_type = ARG_PTR_TO_UNINIT_MEM, 2789c019e2bSYonghong Song .arg2_type = ARG_CONST_SIZE_OR_ZERO, 2792541517cSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 2802541517cSAlexei Starovoitov }; 2812541517cSAlexei Starovoitov 282eb1b6688SDaniel Borkmann BPF_CALL_3(bpf_probe_write_user, void __user *, unsafe_ptr, const void *, src, 283f3694e00SDaniel Borkmann u32, size) 28496ae5227SSargun Dhillon { 28596ae5227SSargun Dhillon /* 28696ae5227SSargun Dhillon * Ensure we're in user context which is safe for the helper to 28796ae5227SSargun Dhillon * run. This helper has no business in a kthread. 28896ae5227SSargun Dhillon * 28996ae5227SSargun Dhillon * access_ok() should prevent writing to non-user memory, but in 29096ae5227SSargun Dhillon * some situations (nommu, temporary switch, etc) access_ok() does 29196ae5227SSargun Dhillon * not provide enough validation, hence the check on KERNEL_DS. 292c7b6f29bSNadav Amit * 293c7b6f29bSNadav Amit * nmi_uaccess_okay() ensures the probe is not run in an interim 294c7b6f29bSNadav Amit * state, when the task or mm are switched. This is specifically 295c7b6f29bSNadav Amit * required to prevent the use of temporary mm. 29696ae5227SSargun Dhillon */ 29796ae5227SSargun Dhillon 29896ae5227SSargun Dhillon if (unlikely(in_interrupt() || 29996ae5227SSargun Dhillon current->flags & (PF_KTHREAD | PF_EXITING))) 30096ae5227SSargun Dhillon return -EPERM; 301db68ce10SAl Viro if (unlikely(uaccess_kernel())) 30296ae5227SSargun Dhillon return -EPERM; 303c7b6f29bSNadav Amit if (unlikely(!nmi_uaccess_okay())) 304c7b6f29bSNadav Amit return -EPERM; 30596ae5227SSargun Dhillon 306eb1b6688SDaniel Borkmann return probe_user_write(unsafe_ptr, src, size); 30796ae5227SSargun Dhillon } 30896ae5227SSargun Dhillon 30996ae5227SSargun Dhillon static const struct bpf_func_proto bpf_probe_write_user_proto = { 31096ae5227SSargun Dhillon .func = bpf_probe_write_user, 31196ae5227SSargun Dhillon .gpl_only = true, 31296ae5227SSargun Dhillon .ret_type = RET_INTEGER, 31396ae5227SSargun Dhillon .arg1_type = ARG_ANYTHING, 31439f19ebbSAlexei Starovoitov .arg2_type = ARG_PTR_TO_MEM, 31539f19ebbSAlexei Starovoitov .arg3_type = ARG_CONST_SIZE, 31696ae5227SSargun Dhillon }; 31796ae5227SSargun Dhillon 31896ae5227SSargun Dhillon static const struct bpf_func_proto *bpf_get_probe_write_proto(void) 31996ae5227SSargun Dhillon { 32096ae5227SSargun Dhillon pr_warn_ratelimited("%s[%d] is installing a program with bpf_probe_write_user helper that may corrupt user memory!", 32196ae5227SSargun Dhillon current->comm, task_pid_nr(current)); 32296ae5227SSargun Dhillon 32396ae5227SSargun Dhillon return &bpf_probe_write_user_proto; 32496ae5227SSargun Dhillon } 32596ae5227SSargun Dhillon 3269c959c86SAlexei Starovoitov /* 3277bda4b40SJohn Fastabend * Only limited trace_printk() conversion specifiers allowed: 3287bda4b40SJohn Fastabend * %d %i %u %x %ld %li %lu %lx %lld %lli %llu %llx %p %s 3299c959c86SAlexei Starovoitov */ 330f3694e00SDaniel Borkmann BPF_CALL_5(bpf_trace_printk, char *, fmt, u32, fmt_size, u64, arg1, 331f3694e00SDaniel Borkmann u64, arg2, u64, arg3) 3329c959c86SAlexei Starovoitov { 3338d3b7dceSAlexei Starovoitov bool str_seen = false; 3349c959c86SAlexei Starovoitov int mod[3] = {}; 3359c959c86SAlexei Starovoitov int fmt_cnt = 0; 3368d3b7dceSAlexei Starovoitov u64 unsafe_addr; 3378d3b7dceSAlexei Starovoitov char buf[64]; 3389c959c86SAlexei Starovoitov int i; 3399c959c86SAlexei Starovoitov 3409c959c86SAlexei Starovoitov /* 3419c959c86SAlexei Starovoitov * bpf_check()->check_func_arg()->check_stack_boundary() 3429c959c86SAlexei Starovoitov * guarantees that fmt points to bpf program stack, 3439c959c86SAlexei Starovoitov * fmt_size bytes of it were initialized and fmt_size > 0 3449c959c86SAlexei Starovoitov */ 3459c959c86SAlexei Starovoitov if (fmt[--fmt_size] != 0) 3469c959c86SAlexei Starovoitov return -EINVAL; 3479c959c86SAlexei Starovoitov 3489c959c86SAlexei Starovoitov /* check format string for allowed specifiers */ 3499c959c86SAlexei Starovoitov for (i = 0; i < fmt_size; i++) { 3509c959c86SAlexei Starovoitov if ((!isprint(fmt[i]) && !isspace(fmt[i])) || !isascii(fmt[i])) 3519c959c86SAlexei Starovoitov return -EINVAL; 3529c959c86SAlexei Starovoitov 3539c959c86SAlexei Starovoitov if (fmt[i] != '%') 3549c959c86SAlexei Starovoitov continue; 3559c959c86SAlexei Starovoitov 3569c959c86SAlexei Starovoitov if (fmt_cnt >= 3) 3579c959c86SAlexei Starovoitov return -EINVAL; 3589c959c86SAlexei Starovoitov 3599c959c86SAlexei Starovoitov /* fmt[i] != 0 && fmt[last] == 0, so we can access fmt[i + 1] */ 3609c959c86SAlexei Starovoitov i++; 3619c959c86SAlexei Starovoitov if (fmt[i] == 'l') { 3629c959c86SAlexei Starovoitov mod[fmt_cnt]++; 3639c959c86SAlexei Starovoitov i++; 3648d3b7dceSAlexei Starovoitov } else if (fmt[i] == 'p' || fmt[i] == 's') { 3659c959c86SAlexei Starovoitov mod[fmt_cnt]++; 3661efb6ee3SMartynas Pumputis /* disallow any further format extensions */ 3671efb6ee3SMartynas Pumputis if (fmt[i + 1] != 0 && 3681efb6ee3SMartynas Pumputis !isspace(fmt[i + 1]) && 3691efb6ee3SMartynas Pumputis !ispunct(fmt[i + 1])) 3709c959c86SAlexei Starovoitov return -EINVAL; 3719c959c86SAlexei Starovoitov fmt_cnt++; 3721efb6ee3SMartynas Pumputis if (fmt[i] == 's') { 3738d3b7dceSAlexei Starovoitov if (str_seen) 3748d3b7dceSAlexei Starovoitov /* allow only one '%s' per fmt string */ 3758d3b7dceSAlexei Starovoitov return -EINVAL; 3768d3b7dceSAlexei Starovoitov str_seen = true; 3778d3b7dceSAlexei Starovoitov 3788d3b7dceSAlexei Starovoitov switch (fmt_cnt) { 3798d3b7dceSAlexei Starovoitov case 1: 380f3694e00SDaniel Borkmann unsafe_addr = arg1; 381f3694e00SDaniel Borkmann arg1 = (long) buf; 3828d3b7dceSAlexei Starovoitov break; 3838d3b7dceSAlexei Starovoitov case 2: 384f3694e00SDaniel Borkmann unsafe_addr = arg2; 385f3694e00SDaniel Borkmann arg2 = (long) buf; 3868d3b7dceSAlexei Starovoitov break; 3878d3b7dceSAlexei Starovoitov case 3: 388f3694e00SDaniel Borkmann unsafe_addr = arg3; 389f3694e00SDaniel Borkmann arg3 = (long) buf; 3908d3b7dceSAlexei Starovoitov break; 3918d3b7dceSAlexei Starovoitov } 3928d3b7dceSAlexei Starovoitov buf[0] = 0; 3938d3b7dceSAlexei Starovoitov strncpy_from_unsafe(buf, 3948d3b7dceSAlexei Starovoitov (void *) (long) unsafe_addr, 3958d3b7dceSAlexei Starovoitov sizeof(buf)); 3968d3b7dceSAlexei Starovoitov } 3979c959c86SAlexei Starovoitov continue; 3989c959c86SAlexei Starovoitov } 3999c959c86SAlexei Starovoitov 4009c959c86SAlexei Starovoitov if (fmt[i] == 'l') { 4019c959c86SAlexei Starovoitov mod[fmt_cnt]++; 4029c959c86SAlexei Starovoitov i++; 4039c959c86SAlexei Starovoitov } 4049c959c86SAlexei Starovoitov 4057bda4b40SJohn Fastabend if (fmt[i] != 'i' && fmt[i] != 'd' && 4067bda4b40SJohn Fastabend fmt[i] != 'u' && fmt[i] != 'x') 4079c959c86SAlexei Starovoitov return -EINVAL; 4089c959c86SAlexei Starovoitov fmt_cnt++; 4099c959c86SAlexei Starovoitov } 4109c959c86SAlexei Starovoitov 41188a5c690SDaniel Borkmann /* Horrid workaround for getting va_list handling working with different 41288a5c690SDaniel Borkmann * argument type combinations generically for 32 and 64 bit archs. 41388a5c690SDaniel Borkmann */ 41488a5c690SDaniel Borkmann #define __BPF_TP_EMIT() __BPF_ARG3_TP() 41588a5c690SDaniel Borkmann #define __BPF_TP(...) \ 416eefa864aSYonghong Song __trace_printk(0 /* Fake ip */, \ 41788a5c690SDaniel Borkmann fmt, ##__VA_ARGS__) 41888a5c690SDaniel Borkmann 41988a5c690SDaniel Borkmann #define __BPF_ARG1_TP(...) \ 42088a5c690SDaniel Borkmann ((mod[0] == 2 || (mod[0] == 1 && __BITS_PER_LONG == 64)) \ 42188a5c690SDaniel Borkmann ? __BPF_TP(arg1, ##__VA_ARGS__) \ 42288a5c690SDaniel Borkmann : ((mod[0] == 1 || (mod[0] == 0 && __BITS_PER_LONG == 32)) \ 42388a5c690SDaniel Borkmann ? __BPF_TP((long)arg1, ##__VA_ARGS__) \ 42488a5c690SDaniel Borkmann : __BPF_TP((u32)arg1, ##__VA_ARGS__))) 42588a5c690SDaniel Borkmann 42688a5c690SDaniel Borkmann #define __BPF_ARG2_TP(...) \ 42788a5c690SDaniel Borkmann ((mod[1] == 2 || (mod[1] == 1 && __BITS_PER_LONG == 64)) \ 42888a5c690SDaniel Borkmann ? __BPF_ARG1_TP(arg2, ##__VA_ARGS__) \ 42988a5c690SDaniel Borkmann : ((mod[1] == 1 || (mod[1] == 0 && __BITS_PER_LONG == 32)) \ 43088a5c690SDaniel Borkmann ? __BPF_ARG1_TP((long)arg2, ##__VA_ARGS__) \ 43188a5c690SDaniel Borkmann : __BPF_ARG1_TP((u32)arg2, ##__VA_ARGS__))) 43288a5c690SDaniel Borkmann 43388a5c690SDaniel Borkmann #define __BPF_ARG3_TP(...) \ 43488a5c690SDaniel Borkmann ((mod[2] == 2 || (mod[2] == 1 && __BITS_PER_LONG == 64)) \ 43588a5c690SDaniel Borkmann ? __BPF_ARG2_TP(arg3, ##__VA_ARGS__) \ 43688a5c690SDaniel Borkmann : ((mod[2] == 1 || (mod[2] == 0 && __BITS_PER_LONG == 32)) \ 43788a5c690SDaniel Borkmann ? __BPF_ARG2_TP((long)arg3, ##__VA_ARGS__) \ 43888a5c690SDaniel Borkmann : __BPF_ARG2_TP((u32)arg3, ##__VA_ARGS__))) 43988a5c690SDaniel Borkmann 44088a5c690SDaniel Borkmann return __BPF_TP_EMIT(); 4419c959c86SAlexei Starovoitov } 4429c959c86SAlexei Starovoitov 4439c959c86SAlexei Starovoitov static const struct bpf_func_proto bpf_trace_printk_proto = { 4449c959c86SAlexei Starovoitov .func = bpf_trace_printk, 4459c959c86SAlexei Starovoitov .gpl_only = true, 4469c959c86SAlexei Starovoitov .ret_type = RET_INTEGER, 44739f19ebbSAlexei Starovoitov .arg1_type = ARG_PTR_TO_MEM, 44839f19ebbSAlexei Starovoitov .arg2_type = ARG_CONST_SIZE, 4499c959c86SAlexei Starovoitov }; 4509c959c86SAlexei Starovoitov 4510756ea3eSAlexei Starovoitov const struct bpf_func_proto *bpf_get_trace_printk_proto(void) 4520756ea3eSAlexei Starovoitov { 4530756ea3eSAlexei Starovoitov /* 4540756ea3eSAlexei Starovoitov * this program might be calling bpf_trace_printk, 4550756ea3eSAlexei Starovoitov * so allocate per-cpu printk buffers 4560756ea3eSAlexei Starovoitov */ 4570756ea3eSAlexei Starovoitov trace_printk_init_buffers(); 4580756ea3eSAlexei Starovoitov 4590756ea3eSAlexei Starovoitov return &bpf_trace_printk_proto; 4600756ea3eSAlexei Starovoitov } 4610756ea3eSAlexei Starovoitov 462908432caSYonghong Song static __always_inline int 463908432caSYonghong Song get_map_perf_counter(struct bpf_map *map, u64 flags, 464908432caSYonghong Song u64 *value, u64 *enabled, u64 *running) 46535578d79SKaixu Xia { 46635578d79SKaixu Xia struct bpf_array *array = container_of(map, struct bpf_array, map); 4676816a7ffSDaniel Borkmann unsigned int cpu = smp_processor_id(); 4686816a7ffSDaniel Borkmann u64 index = flags & BPF_F_INDEX_MASK; 4693b1efb19SDaniel Borkmann struct bpf_event_entry *ee; 47035578d79SKaixu Xia 4716816a7ffSDaniel Borkmann if (unlikely(flags & ~(BPF_F_INDEX_MASK))) 4726816a7ffSDaniel Borkmann return -EINVAL; 4736816a7ffSDaniel Borkmann if (index == BPF_F_CURRENT_CPU) 4746816a7ffSDaniel Borkmann index = cpu; 47535578d79SKaixu Xia if (unlikely(index >= array->map.max_entries)) 47635578d79SKaixu Xia return -E2BIG; 47735578d79SKaixu Xia 4783b1efb19SDaniel Borkmann ee = READ_ONCE(array->ptrs[index]); 4791ca1cc98SDaniel Borkmann if (!ee) 48035578d79SKaixu Xia return -ENOENT; 48135578d79SKaixu Xia 482908432caSYonghong Song return perf_event_read_local(ee->event, value, enabled, running); 483908432caSYonghong Song } 484908432caSYonghong Song 485908432caSYonghong Song BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags) 486908432caSYonghong Song { 487908432caSYonghong Song u64 value = 0; 488908432caSYonghong Song int err; 489908432caSYonghong Song 490908432caSYonghong Song err = get_map_perf_counter(map, flags, &value, NULL, NULL); 49135578d79SKaixu Xia /* 492f91840a3SAlexei Starovoitov * this api is ugly since we miss [-22..-2] range of valid 493f91840a3SAlexei Starovoitov * counter values, but that's uapi 49435578d79SKaixu Xia */ 495f91840a3SAlexei Starovoitov if (err) 496f91840a3SAlexei Starovoitov return err; 497f91840a3SAlexei Starovoitov return value; 49835578d79SKaixu Xia } 49935578d79SKaixu Xia 50062544ce8SAlexei Starovoitov static const struct bpf_func_proto bpf_perf_event_read_proto = { 50135578d79SKaixu Xia .func = bpf_perf_event_read, 5021075ef59SAlexei Starovoitov .gpl_only = true, 50335578d79SKaixu Xia .ret_type = RET_INTEGER, 50435578d79SKaixu Xia .arg1_type = ARG_CONST_MAP_PTR, 50535578d79SKaixu Xia .arg2_type = ARG_ANYTHING, 50635578d79SKaixu Xia }; 50735578d79SKaixu Xia 508908432caSYonghong Song BPF_CALL_4(bpf_perf_event_read_value, struct bpf_map *, map, u64, flags, 509908432caSYonghong Song struct bpf_perf_event_value *, buf, u32, size) 510908432caSYonghong Song { 511908432caSYonghong Song int err = -EINVAL; 512908432caSYonghong Song 513908432caSYonghong Song if (unlikely(size != sizeof(struct bpf_perf_event_value))) 514908432caSYonghong Song goto clear; 515908432caSYonghong Song err = get_map_perf_counter(map, flags, &buf->counter, &buf->enabled, 516908432caSYonghong Song &buf->running); 517908432caSYonghong Song if (unlikely(err)) 518908432caSYonghong Song goto clear; 519908432caSYonghong Song return 0; 520908432caSYonghong Song clear: 521908432caSYonghong Song memset(buf, 0, size); 522908432caSYonghong Song return err; 523908432caSYonghong Song } 524908432caSYonghong Song 525908432caSYonghong Song static const struct bpf_func_proto bpf_perf_event_read_value_proto = { 526908432caSYonghong Song .func = bpf_perf_event_read_value, 527908432caSYonghong Song .gpl_only = true, 528908432caSYonghong Song .ret_type = RET_INTEGER, 529908432caSYonghong Song .arg1_type = ARG_CONST_MAP_PTR, 530908432caSYonghong Song .arg2_type = ARG_ANYTHING, 531908432caSYonghong Song .arg3_type = ARG_PTR_TO_UNINIT_MEM, 532908432caSYonghong Song .arg4_type = ARG_CONST_SIZE, 533908432caSYonghong Song }; 534908432caSYonghong Song 5358e7a3920SDaniel Borkmann static __always_inline u64 5368e7a3920SDaniel Borkmann __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, 537283ca526SDaniel Borkmann u64 flags, struct perf_sample_data *sd) 538a43eec30SAlexei Starovoitov { 539a43eec30SAlexei Starovoitov struct bpf_array *array = container_of(map, struct bpf_array, map); 540d7931330SDaniel Borkmann unsigned int cpu = smp_processor_id(); 5411e33759cSDaniel Borkmann u64 index = flags & BPF_F_INDEX_MASK; 5423b1efb19SDaniel Borkmann struct bpf_event_entry *ee; 543a43eec30SAlexei Starovoitov struct perf_event *event; 544a43eec30SAlexei Starovoitov 5451e33759cSDaniel Borkmann if (index == BPF_F_CURRENT_CPU) 546d7931330SDaniel Borkmann index = cpu; 547a43eec30SAlexei Starovoitov if (unlikely(index >= array->map.max_entries)) 548a43eec30SAlexei Starovoitov return -E2BIG; 549a43eec30SAlexei Starovoitov 5503b1efb19SDaniel Borkmann ee = READ_ONCE(array->ptrs[index]); 5511ca1cc98SDaniel Borkmann if (!ee) 552a43eec30SAlexei Starovoitov return -ENOENT; 553a43eec30SAlexei Starovoitov 5543b1efb19SDaniel Borkmann event = ee->event; 555a43eec30SAlexei Starovoitov if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE || 556a43eec30SAlexei Starovoitov event->attr.config != PERF_COUNT_SW_BPF_OUTPUT)) 557a43eec30SAlexei Starovoitov return -EINVAL; 558a43eec30SAlexei Starovoitov 559d7931330SDaniel Borkmann if (unlikely(event->oncpu != cpu)) 560a43eec30SAlexei Starovoitov return -EOPNOTSUPP; 561a43eec30SAlexei Starovoitov 56256201969SArnaldo Carvalho de Melo return perf_event_output(event, sd, regs); 563a43eec30SAlexei Starovoitov } 564a43eec30SAlexei Starovoitov 5659594dc3cSMatt Mullins /* 5669594dc3cSMatt Mullins * Support executing tracepoints in normal, irq, and nmi context that each call 5679594dc3cSMatt Mullins * bpf_perf_event_output 5689594dc3cSMatt Mullins */ 5699594dc3cSMatt Mullins struct bpf_trace_sample_data { 5709594dc3cSMatt Mullins struct perf_sample_data sds[3]; 5719594dc3cSMatt Mullins }; 5729594dc3cSMatt Mullins 5739594dc3cSMatt Mullins static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds); 5749594dc3cSMatt Mullins static DEFINE_PER_CPU(int, bpf_trace_nest_level); 575f3694e00SDaniel Borkmann BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map, 576f3694e00SDaniel Borkmann u64, flags, void *, data, u64, size) 5778e7a3920SDaniel Borkmann { 5789594dc3cSMatt Mullins struct bpf_trace_sample_data *sds = this_cpu_ptr(&bpf_trace_sds); 5799594dc3cSMatt Mullins int nest_level = this_cpu_inc_return(bpf_trace_nest_level); 5808e7a3920SDaniel Borkmann struct perf_raw_record raw = { 5818e7a3920SDaniel Borkmann .frag = { 5828e7a3920SDaniel Borkmann .size = size, 5838e7a3920SDaniel Borkmann .data = data, 5848e7a3920SDaniel Borkmann }, 5858e7a3920SDaniel Borkmann }; 5869594dc3cSMatt Mullins struct perf_sample_data *sd; 5879594dc3cSMatt Mullins int err; 5888e7a3920SDaniel Borkmann 5899594dc3cSMatt Mullins if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds))) { 5909594dc3cSMatt Mullins err = -EBUSY; 5919594dc3cSMatt Mullins goto out; 5929594dc3cSMatt Mullins } 5939594dc3cSMatt Mullins 5949594dc3cSMatt Mullins sd = &sds->sds[nest_level - 1]; 5959594dc3cSMatt Mullins 5969594dc3cSMatt Mullins if (unlikely(flags & ~(BPF_F_INDEX_MASK))) { 5979594dc3cSMatt Mullins err = -EINVAL; 5989594dc3cSMatt Mullins goto out; 5999594dc3cSMatt Mullins } 6008e7a3920SDaniel Borkmann 601283ca526SDaniel Borkmann perf_sample_data_init(sd, 0, 0); 602283ca526SDaniel Borkmann sd->raw = &raw; 603283ca526SDaniel Borkmann 6049594dc3cSMatt Mullins err = __bpf_perf_event_output(regs, map, flags, sd); 6059594dc3cSMatt Mullins 6069594dc3cSMatt Mullins out: 6079594dc3cSMatt Mullins this_cpu_dec(bpf_trace_nest_level); 6089594dc3cSMatt Mullins return err; 6098e7a3920SDaniel Borkmann } 6108e7a3920SDaniel Borkmann 611a43eec30SAlexei Starovoitov static const struct bpf_func_proto bpf_perf_event_output_proto = { 612a43eec30SAlexei Starovoitov .func = bpf_perf_event_output, 6131075ef59SAlexei Starovoitov .gpl_only = true, 614a43eec30SAlexei Starovoitov .ret_type = RET_INTEGER, 615a43eec30SAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 616a43eec30SAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 617a43eec30SAlexei Starovoitov .arg3_type = ARG_ANYTHING, 61839f19ebbSAlexei Starovoitov .arg4_type = ARG_PTR_TO_MEM, 619a60dd35dSGianluca Borello .arg5_type = ARG_CONST_SIZE_OR_ZERO, 620a43eec30SAlexei Starovoitov }; 621a43eec30SAlexei Starovoitov 622768fb61fSAllan Zhang static DEFINE_PER_CPU(int, bpf_event_output_nest_level); 623768fb61fSAllan Zhang struct bpf_nested_pt_regs { 624768fb61fSAllan Zhang struct pt_regs regs[3]; 625768fb61fSAllan Zhang }; 626768fb61fSAllan Zhang static DEFINE_PER_CPU(struct bpf_nested_pt_regs, bpf_pt_regs); 627768fb61fSAllan Zhang static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_misc_sds); 628bd570ff9SDaniel Borkmann 629555c8a86SDaniel Borkmann u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 630555c8a86SDaniel Borkmann void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy) 631bd570ff9SDaniel Borkmann { 632768fb61fSAllan Zhang int nest_level = this_cpu_inc_return(bpf_event_output_nest_level); 633555c8a86SDaniel Borkmann struct perf_raw_frag frag = { 634555c8a86SDaniel Borkmann .copy = ctx_copy, 635555c8a86SDaniel Borkmann .size = ctx_size, 636555c8a86SDaniel Borkmann .data = ctx, 637555c8a86SDaniel Borkmann }; 638555c8a86SDaniel Borkmann struct perf_raw_record raw = { 639555c8a86SDaniel Borkmann .frag = { 640183fc153SAndrew Morton { 641555c8a86SDaniel Borkmann .next = ctx_size ? &frag : NULL, 642183fc153SAndrew Morton }, 643555c8a86SDaniel Borkmann .size = meta_size, 644555c8a86SDaniel Borkmann .data = meta, 645555c8a86SDaniel Borkmann }, 646555c8a86SDaniel Borkmann }; 647768fb61fSAllan Zhang struct perf_sample_data *sd; 648768fb61fSAllan Zhang struct pt_regs *regs; 649768fb61fSAllan Zhang u64 ret; 650768fb61fSAllan Zhang 651768fb61fSAllan Zhang if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bpf_misc_sds.sds))) { 652768fb61fSAllan Zhang ret = -EBUSY; 653768fb61fSAllan Zhang goto out; 654768fb61fSAllan Zhang } 655768fb61fSAllan Zhang sd = this_cpu_ptr(&bpf_misc_sds.sds[nest_level - 1]); 656768fb61fSAllan Zhang regs = this_cpu_ptr(&bpf_pt_regs.regs[nest_level - 1]); 657bd570ff9SDaniel Borkmann 658bd570ff9SDaniel Borkmann perf_fetch_caller_regs(regs); 659283ca526SDaniel Borkmann perf_sample_data_init(sd, 0, 0); 660283ca526SDaniel Borkmann sd->raw = &raw; 661bd570ff9SDaniel Borkmann 662768fb61fSAllan Zhang ret = __bpf_perf_event_output(regs, map, flags, sd); 663768fb61fSAllan Zhang out: 664768fb61fSAllan Zhang this_cpu_dec(bpf_event_output_nest_level); 665768fb61fSAllan Zhang return ret; 666bd570ff9SDaniel Borkmann } 667bd570ff9SDaniel Borkmann 668f3694e00SDaniel Borkmann BPF_CALL_0(bpf_get_current_task) 669606274c5SAlexei Starovoitov { 670606274c5SAlexei Starovoitov return (long) current; 671606274c5SAlexei Starovoitov } 672606274c5SAlexei Starovoitov 673606274c5SAlexei Starovoitov static const struct bpf_func_proto bpf_get_current_task_proto = { 674606274c5SAlexei Starovoitov .func = bpf_get_current_task, 675606274c5SAlexei Starovoitov .gpl_only = true, 676606274c5SAlexei Starovoitov .ret_type = RET_INTEGER, 677606274c5SAlexei Starovoitov }; 678606274c5SAlexei Starovoitov 679f3694e00SDaniel Borkmann BPF_CALL_2(bpf_current_task_under_cgroup, struct bpf_map *, map, u32, idx) 68060d20f91SSargun Dhillon { 68160d20f91SSargun Dhillon struct bpf_array *array = container_of(map, struct bpf_array, map); 68260d20f91SSargun Dhillon struct cgroup *cgrp; 68360d20f91SSargun Dhillon 68460d20f91SSargun Dhillon if (unlikely(idx >= array->map.max_entries)) 68560d20f91SSargun Dhillon return -E2BIG; 68660d20f91SSargun Dhillon 68760d20f91SSargun Dhillon cgrp = READ_ONCE(array->ptrs[idx]); 68860d20f91SSargun Dhillon if (unlikely(!cgrp)) 68960d20f91SSargun Dhillon return -EAGAIN; 69060d20f91SSargun Dhillon 69160d20f91SSargun Dhillon return task_under_cgroup_hierarchy(current, cgrp); 69260d20f91SSargun Dhillon } 69360d20f91SSargun Dhillon 69460d20f91SSargun Dhillon static const struct bpf_func_proto bpf_current_task_under_cgroup_proto = { 69560d20f91SSargun Dhillon .func = bpf_current_task_under_cgroup, 69660d20f91SSargun Dhillon .gpl_only = false, 69760d20f91SSargun Dhillon .ret_type = RET_INTEGER, 69860d20f91SSargun Dhillon .arg1_type = ARG_CONST_MAP_PTR, 69960d20f91SSargun Dhillon .arg2_type = ARG_ANYTHING, 70060d20f91SSargun Dhillon }; 70160d20f91SSargun Dhillon 7028b401f9eSYonghong Song struct send_signal_irq_work { 7038b401f9eSYonghong Song struct irq_work irq_work; 7048b401f9eSYonghong Song struct task_struct *task; 7058b401f9eSYonghong Song u32 sig; 7068482941fSYonghong Song enum pid_type type; 7078b401f9eSYonghong Song }; 7088b401f9eSYonghong Song 7098b401f9eSYonghong Song static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work); 7108b401f9eSYonghong Song 7118b401f9eSYonghong Song static void do_bpf_send_signal(struct irq_work *entry) 7128b401f9eSYonghong Song { 7138b401f9eSYonghong Song struct send_signal_irq_work *work; 7148b401f9eSYonghong Song 7158b401f9eSYonghong Song work = container_of(entry, struct send_signal_irq_work, irq_work); 7168482941fSYonghong Song group_send_sig_info(work->sig, SEND_SIG_PRIV, work->task, work->type); 7178b401f9eSYonghong Song } 7188b401f9eSYonghong Song 7198482941fSYonghong Song static int bpf_send_signal_common(u32 sig, enum pid_type type) 7208b401f9eSYonghong Song { 7218b401f9eSYonghong Song struct send_signal_irq_work *work = NULL; 7228b401f9eSYonghong Song 7238b401f9eSYonghong Song /* Similar to bpf_probe_write_user, task needs to be 7248b401f9eSYonghong Song * in a sound condition and kernel memory access be 7258b401f9eSYonghong Song * permitted in order to send signal to the current 7268b401f9eSYonghong Song * task. 7278b401f9eSYonghong Song */ 7288b401f9eSYonghong Song if (unlikely(current->flags & (PF_KTHREAD | PF_EXITING))) 7298b401f9eSYonghong Song return -EPERM; 7308b401f9eSYonghong Song if (unlikely(uaccess_kernel())) 7318b401f9eSYonghong Song return -EPERM; 7328b401f9eSYonghong Song if (unlikely(!nmi_uaccess_okay())) 7338b401f9eSYonghong Song return -EPERM; 7348b401f9eSYonghong Song 7358b401f9eSYonghong Song if (in_nmi()) { 736e1afb702SYonghong Song /* Do an early check on signal validity. Otherwise, 737e1afb702SYonghong Song * the error is lost in deferred irq_work. 738e1afb702SYonghong Song */ 739e1afb702SYonghong Song if (unlikely(!valid_signal(sig))) 740e1afb702SYonghong Song return -EINVAL; 741e1afb702SYonghong Song 7428b401f9eSYonghong Song work = this_cpu_ptr(&send_signal_work); 743153bedbaSFrederic Weisbecker if (atomic_read(&work->irq_work.flags) & IRQ_WORK_BUSY) 7448b401f9eSYonghong Song return -EBUSY; 7458b401f9eSYonghong Song 7468b401f9eSYonghong Song /* Add the current task, which is the target of sending signal, 7478b401f9eSYonghong Song * to the irq_work. The current task may change when queued 7488b401f9eSYonghong Song * irq works get executed. 7498b401f9eSYonghong Song */ 7508b401f9eSYonghong Song work->task = current; 7518b401f9eSYonghong Song work->sig = sig; 7528482941fSYonghong Song work->type = type; 7538b401f9eSYonghong Song irq_work_queue(&work->irq_work); 7548b401f9eSYonghong Song return 0; 7558b401f9eSYonghong Song } 7568b401f9eSYonghong Song 7578482941fSYonghong Song return group_send_sig_info(sig, SEND_SIG_PRIV, current, type); 7588482941fSYonghong Song } 7598482941fSYonghong Song 7608482941fSYonghong Song BPF_CALL_1(bpf_send_signal, u32, sig) 7618482941fSYonghong Song { 7628482941fSYonghong Song return bpf_send_signal_common(sig, PIDTYPE_TGID); 7638b401f9eSYonghong Song } 7648b401f9eSYonghong Song 7658b401f9eSYonghong Song static const struct bpf_func_proto bpf_send_signal_proto = { 7668b401f9eSYonghong Song .func = bpf_send_signal, 7678b401f9eSYonghong Song .gpl_only = false, 7688b401f9eSYonghong Song .ret_type = RET_INTEGER, 7698b401f9eSYonghong Song .arg1_type = ARG_ANYTHING, 7708b401f9eSYonghong Song }; 7718b401f9eSYonghong Song 7728482941fSYonghong Song BPF_CALL_1(bpf_send_signal_thread, u32, sig) 7738482941fSYonghong Song { 7748482941fSYonghong Song return bpf_send_signal_common(sig, PIDTYPE_PID); 7758482941fSYonghong Song } 7768482941fSYonghong Song 7778482941fSYonghong Song static const struct bpf_func_proto bpf_send_signal_thread_proto = { 7788482941fSYonghong Song .func = bpf_send_signal_thread, 7798482941fSYonghong Song .gpl_only = false, 7808482941fSYonghong Song .ret_type = RET_INTEGER, 7818482941fSYonghong Song .arg1_type = ARG_ANYTHING, 7828482941fSYonghong Song }; 7838482941fSYonghong Song 7845e43f899SAndrey Ignatov static const struct bpf_func_proto * 7855e43f899SAndrey Ignatov tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 7862541517cSAlexei Starovoitov { 7872541517cSAlexei Starovoitov switch (func_id) { 7882541517cSAlexei Starovoitov case BPF_FUNC_map_lookup_elem: 7892541517cSAlexei Starovoitov return &bpf_map_lookup_elem_proto; 7902541517cSAlexei Starovoitov case BPF_FUNC_map_update_elem: 7912541517cSAlexei Starovoitov return &bpf_map_update_elem_proto; 7922541517cSAlexei Starovoitov case BPF_FUNC_map_delete_elem: 7932541517cSAlexei Starovoitov return &bpf_map_delete_elem_proto; 79402a8c817SAlban Crequy case BPF_FUNC_map_push_elem: 79502a8c817SAlban Crequy return &bpf_map_push_elem_proto; 79602a8c817SAlban Crequy case BPF_FUNC_map_pop_elem: 79702a8c817SAlban Crequy return &bpf_map_pop_elem_proto; 79802a8c817SAlban Crequy case BPF_FUNC_map_peek_elem: 79902a8c817SAlban Crequy return &bpf_map_peek_elem_proto; 800d9847d31SAlexei Starovoitov case BPF_FUNC_ktime_get_ns: 801d9847d31SAlexei Starovoitov return &bpf_ktime_get_ns_proto; 80204fd61abSAlexei Starovoitov case BPF_FUNC_tail_call: 80304fd61abSAlexei Starovoitov return &bpf_tail_call_proto; 804ffeedafbSAlexei Starovoitov case BPF_FUNC_get_current_pid_tgid: 805ffeedafbSAlexei Starovoitov return &bpf_get_current_pid_tgid_proto; 806606274c5SAlexei Starovoitov case BPF_FUNC_get_current_task: 807606274c5SAlexei Starovoitov return &bpf_get_current_task_proto; 808ffeedafbSAlexei Starovoitov case BPF_FUNC_get_current_uid_gid: 809ffeedafbSAlexei Starovoitov return &bpf_get_current_uid_gid_proto; 810ffeedafbSAlexei Starovoitov case BPF_FUNC_get_current_comm: 811ffeedafbSAlexei Starovoitov return &bpf_get_current_comm_proto; 8129c959c86SAlexei Starovoitov case BPF_FUNC_trace_printk: 8130756ea3eSAlexei Starovoitov return bpf_get_trace_printk_proto(); 814ab1973d3SAlexei Starovoitov case BPF_FUNC_get_smp_processor_id: 815ab1973d3SAlexei Starovoitov return &bpf_get_smp_processor_id_proto; 8162d0e30c3SDaniel Borkmann case BPF_FUNC_get_numa_node_id: 8172d0e30c3SDaniel Borkmann return &bpf_get_numa_node_id_proto; 81835578d79SKaixu Xia case BPF_FUNC_perf_event_read: 81935578d79SKaixu Xia return &bpf_perf_event_read_proto; 82096ae5227SSargun Dhillon case BPF_FUNC_probe_write_user: 82196ae5227SSargun Dhillon return bpf_get_probe_write_proto(); 82260d20f91SSargun Dhillon case BPF_FUNC_current_task_under_cgroup: 82360d20f91SSargun Dhillon return &bpf_current_task_under_cgroup_proto; 8248937bd80SAlexei Starovoitov case BPF_FUNC_get_prandom_u32: 8258937bd80SAlexei Starovoitov return &bpf_get_prandom_u32_proto; 8266ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read_user: 8276ae08ae3SDaniel Borkmann return &bpf_probe_read_user_proto; 8286ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read_kernel: 8296ae08ae3SDaniel Borkmann return &bpf_probe_read_kernel_proto; 8306ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read: 8316ae08ae3SDaniel Borkmann return &bpf_probe_read_compat_proto; 8326ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read_user_str: 8336ae08ae3SDaniel Borkmann return &bpf_probe_read_user_str_proto; 8346ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read_kernel_str: 8356ae08ae3SDaniel Borkmann return &bpf_probe_read_kernel_str_proto; 836a5e8c070SGianluca Borello case BPF_FUNC_probe_read_str: 8376ae08ae3SDaniel Borkmann return &bpf_probe_read_compat_str_proto; 83834ea38caSYonghong Song #ifdef CONFIG_CGROUPS 839bf6fa2c8SYonghong Song case BPF_FUNC_get_current_cgroup_id: 840bf6fa2c8SYonghong Song return &bpf_get_current_cgroup_id_proto; 84134ea38caSYonghong Song #endif 8428b401f9eSYonghong Song case BPF_FUNC_send_signal: 8438b401f9eSYonghong Song return &bpf_send_signal_proto; 8448482941fSYonghong Song case BPF_FUNC_send_signal_thread: 8458482941fSYonghong Song return &bpf_send_signal_thread_proto; 846b80b033bSSong Liu case BPF_FUNC_perf_event_read_value: 847b80b033bSSong Liu return &bpf_perf_event_read_value_proto; 8489fd82b61SAlexei Starovoitov default: 8499fd82b61SAlexei Starovoitov return NULL; 8509fd82b61SAlexei Starovoitov } 8519fd82b61SAlexei Starovoitov } 8529fd82b61SAlexei Starovoitov 8535e43f899SAndrey Ignatov static const struct bpf_func_proto * 8545e43f899SAndrey Ignatov kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 8559fd82b61SAlexei Starovoitov { 8569fd82b61SAlexei Starovoitov switch (func_id) { 857a43eec30SAlexei Starovoitov case BPF_FUNC_perf_event_output: 858a43eec30SAlexei Starovoitov return &bpf_perf_event_output_proto; 859d5a3b1f6SAlexei Starovoitov case BPF_FUNC_get_stackid: 860d5a3b1f6SAlexei Starovoitov return &bpf_get_stackid_proto; 861c195651eSYonghong Song case BPF_FUNC_get_stack: 862c195651eSYonghong Song return &bpf_get_stack_proto; 8639802d865SJosef Bacik #ifdef CONFIG_BPF_KPROBE_OVERRIDE 8649802d865SJosef Bacik case BPF_FUNC_override_return: 8659802d865SJosef Bacik return &bpf_override_return_proto; 8669802d865SJosef Bacik #endif 8672541517cSAlexei Starovoitov default: 8685e43f899SAndrey Ignatov return tracing_func_proto(func_id, prog); 8692541517cSAlexei Starovoitov } 8702541517cSAlexei Starovoitov } 8712541517cSAlexei Starovoitov 8722541517cSAlexei Starovoitov /* bpf+kprobe programs can access fields of 'struct pt_regs' */ 87319de99f7SAlexei Starovoitov static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type type, 8745e43f899SAndrey Ignatov const struct bpf_prog *prog, 87523994631SYonghong Song struct bpf_insn_access_aux *info) 8762541517cSAlexei Starovoitov { 8772541517cSAlexei Starovoitov if (off < 0 || off >= sizeof(struct pt_regs)) 8782541517cSAlexei Starovoitov return false; 8792541517cSAlexei Starovoitov if (type != BPF_READ) 8802541517cSAlexei Starovoitov return false; 8812541517cSAlexei Starovoitov if (off % size != 0) 8822541517cSAlexei Starovoitov return false; 8832d071c64SDaniel Borkmann /* 8842d071c64SDaniel Borkmann * Assertion for 32 bit to make sure last 8 byte access 8852d071c64SDaniel Borkmann * (BPF_DW) to the last 4 byte member is disallowed. 8862d071c64SDaniel Borkmann */ 8872d071c64SDaniel Borkmann if (off + size > sizeof(struct pt_regs)) 8882d071c64SDaniel Borkmann return false; 8892d071c64SDaniel Borkmann 8902541517cSAlexei Starovoitov return true; 8912541517cSAlexei Starovoitov } 8922541517cSAlexei Starovoitov 8937de16e3aSJakub Kicinski const struct bpf_verifier_ops kprobe_verifier_ops = { 8942541517cSAlexei Starovoitov .get_func_proto = kprobe_prog_func_proto, 8952541517cSAlexei Starovoitov .is_valid_access = kprobe_prog_is_valid_access, 8962541517cSAlexei Starovoitov }; 8972541517cSAlexei Starovoitov 8987de16e3aSJakub Kicinski const struct bpf_prog_ops kprobe_prog_ops = { 8997de16e3aSJakub Kicinski }; 9007de16e3aSJakub Kicinski 901f3694e00SDaniel Borkmann BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map, 902f3694e00SDaniel Borkmann u64, flags, void *, data, u64, size) 9039940d67cSAlexei Starovoitov { 904f3694e00SDaniel Borkmann struct pt_regs *regs = *(struct pt_regs **)tp_buff; 905f3694e00SDaniel Borkmann 9069940d67cSAlexei Starovoitov /* 9079940d67cSAlexei Starovoitov * r1 points to perf tracepoint buffer where first 8 bytes are hidden 9089940d67cSAlexei Starovoitov * from bpf program and contain a pointer to 'struct pt_regs'. Fetch it 909f3694e00SDaniel Borkmann * from there and call the same bpf_perf_event_output() helper inline. 9109940d67cSAlexei Starovoitov */ 911f3694e00SDaniel Borkmann return ____bpf_perf_event_output(regs, map, flags, data, size); 9129940d67cSAlexei Starovoitov } 9139940d67cSAlexei Starovoitov 9149940d67cSAlexei Starovoitov static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { 9159940d67cSAlexei Starovoitov .func = bpf_perf_event_output_tp, 9169940d67cSAlexei Starovoitov .gpl_only = true, 9179940d67cSAlexei Starovoitov .ret_type = RET_INTEGER, 9189940d67cSAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 9199940d67cSAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 9209940d67cSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 92139f19ebbSAlexei Starovoitov .arg4_type = ARG_PTR_TO_MEM, 922a60dd35dSGianluca Borello .arg5_type = ARG_CONST_SIZE_OR_ZERO, 9239940d67cSAlexei Starovoitov }; 9249940d67cSAlexei Starovoitov 925f3694e00SDaniel Borkmann BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map, 926f3694e00SDaniel Borkmann u64, flags) 9279940d67cSAlexei Starovoitov { 928f3694e00SDaniel Borkmann struct pt_regs *regs = *(struct pt_regs **)tp_buff; 9299940d67cSAlexei Starovoitov 930f3694e00SDaniel Borkmann /* 931f3694e00SDaniel Borkmann * Same comment as in bpf_perf_event_output_tp(), only that this time 932f3694e00SDaniel Borkmann * the other helper's function body cannot be inlined due to being 933f3694e00SDaniel Borkmann * external, thus we need to call raw helper function. 934f3694e00SDaniel Borkmann */ 935f3694e00SDaniel Borkmann return bpf_get_stackid((unsigned long) regs, (unsigned long) map, 936f3694e00SDaniel Borkmann flags, 0, 0); 9379940d67cSAlexei Starovoitov } 9389940d67cSAlexei Starovoitov 9399940d67cSAlexei Starovoitov static const struct bpf_func_proto bpf_get_stackid_proto_tp = { 9409940d67cSAlexei Starovoitov .func = bpf_get_stackid_tp, 9419940d67cSAlexei Starovoitov .gpl_only = true, 9429940d67cSAlexei Starovoitov .ret_type = RET_INTEGER, 9439940d67cSAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 9449940d67cSAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 9459940d67cSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 9469940d67cSAlexei Starovoitov }; 9479940d67cSAlexei Starovoitov 948c195651eSYonghong Song BPF_CALL_4(bpf_get_stack_tp, void *, tp_buff, void *, buf, u32, size, 949c195651eSYonghong Song u64, flags) 950c195651eSYonghong Song { 951c195651eSYonghong Song struct pt_regs *regs = *(struct pt_regs **)tp_buff; 952c195651eSYonghong Song 953c195651eSYonghong Song return bpf_get_stack((unsigned long) regs, (unsigned long) buf, 954c195651eSYonghong Song (unsigned long) size, flags, 0); 955c195651eSYonghong Song } 956c195651eSYonghong Song 957c195651eSYonghong Song static const struct bpf_func_proto bpf_get_stack_proto_tp = { 958c195651eSYonghong Song .func = bpf_get_stack_tp, 959c195651eSYonghong Song .gpl_only = true, 960c195651eSYonghong Song .ret_type = RET_INTEGER, 961c195651eSYonghong Song .arg1_type = ARG_PTR_TO_CTX, 962c195651eSYonghong Song .arg2_type = ARG_PTR_TO_UNINIT_MEM, 963c195651eSYonghong Song .arg3_type = ARG_CONST_SIZE_OR_ZERO, 964c195651eSYonghong Song .arg4_type = ARG_ANYTHING, 965c195651eSYonghong Song }; 966c195651eSYonghong Song 9675e43f899SAndrey Ignatov static const struct bpf_func_proto * 9685e43f899SAndrey Ignatov tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 9699fd82b61SAlexei Starovoitov { 9709fd82b61SAlexei Starovoitov switch (func_id) { 9719fd82b61SAlexei Starovoitov case BPF_FUNC_perf_event_output: 9729940d67cSAlexei Starovoitov return &bpf_perf_event_output_proto_tp; 9739fd82b61SAlexei Starovoitov case BPF_FUNC_get_stackid: 9749940d67cSAlexei Starovoitov return &bpf_get_stackid_proto_tp; 975c195651eSYonghong Song case BPF_FUNC_get_stack: 976c195651eSYonghong Song return &bpf_get_stack_proto_tp; 9779fd82b61SAlexei Starovoitov default: 9785e43f899SAndrey Ignatov return tracing_func_proto(func_id, prog); 9799fd82b61SAlexei Starovoitov } 9809fd82b61SAlexei Starovoitov } 9819fd82b61SAlexei Starovoitov 98219de99f7SAlexei Starovoitov static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type, 9835e43f899SAndrey Ignatov const struct bpf_prog *prog, 98423994631SYonghong Song struct bpf_insn_access_aux *info) 9859fd82b61SAlexei Starovoitov { 9869fd82b61SAlexei Starovoitov if (off < sizeof(void *) || off >= PERF_MAX_TRACE_SIZE) 9879fd82b61SAlexei Starovoitov return false; 9889fd82b61SAlexei Starovoitov if (type != BPF_READ) 9899fd82b61SAlexei Starovoitov return false; 9909fd82b61SAlexei Starovoitov if (off % size != 0) 9919fd82b61SAlexei Starovoitov return false; 9922d071c64SDaniel Borkmann 9932d071c64SDaniel Borkmann BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(__u64)); 9949fd82b61SAlexei Starovoitov return true; 9959fd82b61SAlexei Starovoitov } 9969fd82b61SAlexei Starovoitov 9977de16e3aSJakub Kicinski const struct bpf_verifier_ops tracepoint_verifier_ops = { 9989fd82b61SAlexei Starovoitov .get_func_proto = tp_prog_func_proto, 9999fd82b61SAlexei Starovoitov .is_valid_access = tp_prog_is_valid_access, 10009fd82b61SAlexei Starovoitov }; 10019fd82b61SAlexei Starovoitov 10027de16e3aSJakub Kicinski const struct bpf_prog_ops tracepoint_prog_ops = { 10037de16e3aSJakub Kicinski }; 10047de16e3aSJakub Kicinski 1005f005afedSYonghong Song BPF_CALL_3(bpf_perf_prog_read_value, struct bpf_perf_event_data_kern *, ctx, 1006f005afedSYonghong Song struct bpf_perf_event_value *, buf, u32, size) 1007f005afedSYonghong Song { 1008f005afedSYonghong Song int err = -EINVAL; 1009f005afedSYonghong Song 1010f005afedSYonghong Song if (unlikely(size != sizeof(struct bpf_perf_event_value))) 1011f005afedSYonghong Song goto clear; 1012f005afedSYonghong Song err = perf_event_read_local(ctx->event, &buf->counter, &buf->enabled, 1013f005afedSYonghong Song &buf->running); 1014f005afedSYonghong Song if (unlikely(err)) 1015f005afedSYonghong Song goto clear; 1016f005afedSYonghong Song return 0; 1017f005afedSYonghong Song clear: 1018f005afedSYonghong Song memset(buf, 0, size); 1019f005afedSYonghong Song return err; 1020f005afedSYonghong Song } 1021f005afedSYonghong Song 1022f005afedSYonghong Song static const struct bpf_func_proto bpf_perf_prog_read_value_proto = { 1023f005afedSYonghong Song .func = bpf_perf_prog_read_value, 1024f005afedSYonghong Song .gpl_only = true, 1025f005afedSYonghong Song .ret_type = RET_INTEGER, 1026f005afedSYonghong Song .arg1_type = ARG_PTR_TO_CTX, 1027f005afedSYonghong Song .arg2_type = ARG_PTR_TO_UNINIT_MEM, 1028f005afedSYonghong Song .arg3_type = ARG_CONST_SIZE, 1029f005afedSYonghong Song }; 1030f005afedSYonghong Song 1031fff7b643SDaniel Xu BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx, 1032fff7b643SDaniel Xu void *, buf, u32, size, u64, flags) 1033fff7b643SDaniel Xu { 1034fff7b643SDaniel Xu #ifndef CONFIG_X86 1035fff7b643SDaniel Xu return -ENOENT; 1036fff7b643SDaniel Xu #else 1037fff7b643SDaniel Xu static const u32 br_entry_size = sizeof(struct perf_branch_entry); 1038fff7b643SDaniel Xu struct perf_branch_stack *br_stack = ctx->data->br_stack; 1039fff7b643SDaniel Xu u32 to_copy; 1040fff7b643SDaniel Xu 1041fff7b643SDaniel Xu if (unlikely(flags & ~BPF_F_GET_BRANCH_RECORDS_SIZE)) 1042fff7b643SDaniel Xu return -EINVAL; 1043fff7b643SDaniel Xu 1044fff7b643SDaniel Xu if (unlikely(!br_stack)) 1045fff7b643SDaniel Xu return -EINVAL; 1046fff7b643SDaniel Xu 1047fff7b643SDaniel Xu if (flags & BPF_F_GET_BRANCH_RECORDS_SIZE) 1048fff7b643SDaniel Xu return br_stack->nr * br_entry_size; 1049fff7b643SDaniel Xu 1050fff7b643SDaniel Xu if (!buf || (size % br_entry_size != 0)) 1051fff7b643SDaniel Xu return -EINVAL; 1052fff7b643SDaniel Xu 1053fff7b643SDaniel Xu to_copy = min_t(u32, br_stack->nr * br_entry_size, size); 1054fff7b643SDaniel Xu memcpy(buf, br_stack->entries, to_copy); 1055fff7b643SDaniel Xu 1056fff7b643SDaniel Xu return to_copy; 1057fff7b643SDaniel Xu #endif 1058fff7b643SDaniel Xu } 1059fff7b643SDaniel Xu 1060fff7b643SDaniel Xu static const struct bpf_func_proto bpf_read_branch_records_proto = { 1061fff7b643SDaniel Xu .func = bpf_read_branch_records, 1062fff7b643SDaniel Xu .gpl_only = true, 1063fff7b643SDaniel Xu .ret_type = RET_INTEGER, 1064fff7b643SDaniel Xu .arg1_type = ARG_PTR_TO_CTX, 1065fff7b643SDaniel Xu .arg2_type = ARG_PTR_TO_MEM_OR_NULL, 1066fff7b643SDaniel Xu .arg3_type = ARG_CONST_SIZE_OR_ZERO, 1067fff7b643SDaniel Xu .arg4_type = ARG_ANYTHING, 1068fff7b643SDaniel Xu }; 1069fff7b643SDaniel Xu 10705e43f899SAndrey Ignatov static const struct bpf_func_proto * 10715e43f899SAndrey Ignatov pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1072f005afedSYonghong Song { 1073f005afedSYonghong Song switch (func_id) { 1074f005afedSYonghong Song case BPF_FUNC_perf_event_output: 1075f005afedSYonghong Song return &bpf_perf_event_output_proto_tp; 1076f005afedSYonghong Song case BPF_FUNC_get_stackid: 1077f005afedSYonghong Song return &bpf_get_stackid_proto_tp; 1078c195651eSYonghong Song case BPF_FUNC_get_stack: 1079c195651eSYonghong Song return &bpf_get_stack_proto_tp; 1080f005afedSYonghong Song case BPF_FUNC_perf_prog_read_value: 1081f005afedSYonghong Song return &bpf_perf_prog_read_value_proto; 1082fff7b643SDaniel Xu case BPF_FUNC_read_branch_records: 1083fff7b643SDaniel Xu return &bpf_read_branch_records_proto; 1084f005afedSYonghong Song default: 10855e43f899SAndrey Ignatov return tracing_func_proto(func_id, prog); 1086f005afedSYonghong Song } 1087f005afedSYonghong Song } 1088f005afedSYonghong Song 1089c4f6699dSAlexei Starovoitov /* 1090c4f6699dSAlexei Starovoitov * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp 1091c4f6699dSAlexei Starovoitov * to avoid potential recursive reuse issue when/if tracepoints are added 10929594dc3cSMatt Mullins * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack. 10939594dc3cSMatt Mullins * 10949594dc3cSMatt Mullins * Since raw tracepoints run despite bpf_prog_active, support concurrent usage 10959594dc3cSMatt Mullins * in normal, irq, and nmi context. 1096c4f6699dSAlexei Starovoitov */ 10979594dc3cSMatt Mullins struct bpf_raw_tp_regs { 10989594dc3cSMatt Mullins struct pt_regs regs[3]; 10999594dc3cSMatt Mullins }; 11009594dc3cSMatt Mullins static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs); 11019594dc3cSMatt Mullins static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level); 11029594dc3cSMatt Mullins static struct pt_regs *get_bpf_raw_tp_regs(void) 11039594dc3cSMatt Mullins { 11049594dc3cSMatt Mullins struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs); 11059594dc3cSMatt Mullins int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level); 11069594dc3cSMatt Mullins 11079594dc3cSMatt Mullins if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(tp_regs->regs))) { 11089594dc3cSMatt Mullins this_cpu_dec(bpf_raw_tp_nest_level); 11099594dc3cSMatt Mullins return ERR_PTR(-EBUSY); 11109594dc3cSMatt Mullins } 11119594dc3cSMatt Mullins 11129594dc3cSMatt Mullins return &tp_regs->regs[nest_level - 1]; 11139594dc3cSMatt Mullins } 11149594dc3cSMatt Mullins 11159594dc3cSMatt Mullins static void put_bpf_raw_tp_regs(void) 11169594dc3cSMatt Mullins { 11179594dc3cSMatt Mullins this_cpu_dec(bpf_raw_tp_nest_level); 11189594dc3cSMatt Mullins } 11199594dc3cSMatt Mullins 1120c4f6699dSAlexei Starovoitov BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args, 1121c4f6699dSAlexei Starovoitov struct bpf_map *, map, u64, flags, void *, data, u64, size) 1122c4f6699dSAlexei Starovoitov { 11239594dc3cSMatt Mullins struct pt_regs *regs = get_bpf_raw_tp_regs(); 11249594dc3cSMatt Mullins int ret; 11259594dc3cSMatt Mullins 11269594dc3cSMatt Mullins if (IS_ERR(regs)) 11279594dc3cSMatt Mullins return PTR_ERR(regs); 1128c4f6699dSAlexei Starovoitov 1129c4f6699dSAlexei Starovoitov perf_fetch_caller_regs(regs); 11309594dc3cSMatt Mullins ret = ____bpf_perf_event_output(regs, map, flags, data, size); 11319594dc3cSMatt Mullins 11329594dc3cSMatt Mullins put_bpf_raw_tp_regs(); 11339594dc3cSMatt Mullins return ret; 1134c4f6699dSAlexei Starovoitov } 1135c4f6699dSAlexei Starovoitov 1136c4f6699dSAlexei Starovoitov static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { 1137c4f6699dSAlexei Starovoitov .func = bpf_perf_event_output_raw_tp, 1138c4f6699dSAlexei Starovoitov .gpl_only = true, 1139c4f6699dSAlexei Starovoitov .ret_type = RET_INTEGER, 1140c4f6699dSAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 1141c4f6699dSAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 1142c4f6699dSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 1143c4f6699dSAlexei Starovoitov .arg4_type = ARG_PTR_TO_MEM, 1144c4f6699dSAlexei Starovoitov .arg5_type = ARG_CONST_SIZE_OR_ZERO, 1145c4f6699dSAlexei Starovoitov }; 1146c4f6699dSAlexei Starovoitov 1147a7658e1aSAlexei Starovoitov extern const struct bpf_func_proto bpf_skb_output_proto; 1148a7658e1aSAlexei Starovoitov 1149c4f6699dSAlexei Starovoitov BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args, 1150c4f6699dSAlexei Starovoitov struct bpf_map *, map, u64, flags) 1151c4f6699dSAlexei Starovoitov { 11529594dc3cSMatt Mullins struct pt_regs *regs = get_bpf_raw_tp_regs(); 11539594dc3cSMatt Mullins int ret; 11549594dc3cSMatt Mullins 11559594dc3cSMatt Mullins if (IS_ERR(regs)) 11569594dc3cSMatt Mullins return PTR_ERR(regs); 1157c4f6699dSAlexei Starovoitov 1158c4f6699dSAlexei Starovoitov perf_fetch_caller_regs(regs); 1159c4f6699dSAlexei Starovoitov /* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */ 11609594dc3cSMatt Mullins ret = bpf_get_stackid((unsigned long) regs, (unsigned long) map, 1161c4f6699dSAlexei Starovoitov flags, 0, 0); 11629594dc3cSMatt Mullins put_bpf_raw_tp_regs(); 11639594dc3cSMatt Mullins return ret; 1164c4f6699dSAlexei Starovoitov } 1165c4f6699dSAlexei Starovoitov 1166c4f6699dSAlexei Starovoitov static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = { 1167c4f6699dSAlexei Starovoitov .func = bpf_get_stackid_raw_tp, 1168c4f6699dSAlexei Starovoitov .gpl_only = true, 1169c4f6699dSAlexei Starovoitov .ret_type = RET_INTEGER, 1170c4f6699dSAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 1171c4f6699dSAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 1172c4f6699dSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 1173c4f6699dSAlexei Starovoitov }; 1174c4f6699dSAlexei Starovoitov 1175c195651eSYonghong Song BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args, 1176c195651eSYonghong Song void *, buf, u32, size, u64, flags) 1177c195651eSYonghong Song { 11789594dc3cSMatt Mullins struct pt_regs *regs = get_bpf_raw_tp_regs(); 11799594dc3cSMatt Mullins int ret; 11809594dc3cSMatt Mullins 11819594dc3cSMatt Mullins if (IS_ERR(regs)) 11829594dc3cSMatt Mullins return PTR_ERR(regs); 1183c195651eSYonghong Song 1184c195651eSYonghong Song perf_fetch_caller_regs(regs); 11859594dc3cSMatt Mullins ret = bpf_get_stack((unsigned long) regs, (unsigned long) buf, 1186c195651eSYonghong Song (unsigned long) size, flags, 0); 11879594dc3cSMatt Mullins put_bpf_raw_tp_regs(); 11889594dc3cSMatt Mullins return ret; 1189c195651eSYonghong Song } 1190c195651eSYonghong Song 1191c195651eSYonghong Song static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { 1192c195651eSYonghong Song .func = bpf_get_stack_raw_tp, 1193c195651eSYonghong Song .gpl_only = true, 1194c195651eSYonghong Song .ret_type = RET_INTEGER, 1195c195651eSYonghong Song .arg1_type = ARG_PTR_TO_CTX, 1196c195651eSYonghong Song .arg2_type = ARG_PTR_TO_MEM, 1197c195651eSYonghong Song .arg3_type = ARG_CONST_SIZE_OR_ZERO, 1198c195651eSYonghong Song .arg4_type = ARG_ANYTHING, 1199c195651eSYonghong Song }; 1200c195651eSYonghong Song 12015e43f899SAndrey Ignatov static const struct bpf_func_proto * 12025e43f899SAndrey Ignatov raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1203c4f6699dSAlexei Starovoitov { 1204c4f6699dSAlexei Starovoitov switch (func_id) { 1205c4f6699dSAlexei Starovoitov case BPF_FUNC_perf_event_output: 1206c4f6699dSAlexei Starovoitov return &bpf_perf_event_output_proto_raw_tp; 1207c4f6699dSAlexei Starovoitov case BPF_FUNC_get_stackid: 1208c4f6699dSAlexei Starovoitov return &bpf_get_stackid_proto_raw_tp; 1209c195651eSYonghong Song case BPF_FUNC_get_stack: 1210c195651eSYonghong Song return &bpf_get_stack_proto_raw_tp; 1211c4f6699dSAlexei Starovoitov default: 12125e43f899SAndrey Ignatov return tracing_func_proto(func_id, prog); 1213c4f6699dSAlexei Starovoitov } 1214c4f6699dSAlexei Starovoitov } 1215c4f6699dSAlexei Starovoitov 1216f1b9509cSAlexei Starovoitov static const struct bpf_func_proto * 1217f1b9509cSAlexei Starovoitov tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1218f1b9509cSAlexei Starovoitov { 1219f1b9509cSAlexei Starovoitov switch (func_id) { 1220f1b9509cSAlexei Starovoitov #ifdef CONFIG_NET 1221f1b9509cSAlexei Starovoitov case BPF_FUNC_skb_output: 1222f1b9509cSAlexei Starovoitov return &bpf_skb_output_proto; 1223f1b9509cSAlexei Starovoitov #endif 1224f1b9509cSAlexei Starovoitov default: 1225f1b9509cSAlexei Starovoitov return raw_tp_prog_func_proto(func_id, prog); 1226f1b9509cSAlexei Starovoitov } 1227f1b9509cSAlexei Starovoitov } 1228f1b9509cSAlexei Starovoitov 1229c4f6699dSAlexei Starovoitov static bool raw_tp_prog_is_valid_access(int off, int size, 1230c4f6699dSAlexei Starovoitov enum bpf_access_type type, 12315e43f899SAndrey Ignatov const struct bpf_prog *prog, 1232c4f6699dSAlexei Starovoitov struct bpf_insn_access_aux *info) 1233c4f6699dSAlexei Starovoitov { 1234f1b9509cSAlexei Starovoitov if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 1235c4f6699dSAlexei Starovoitov return false; 1236c4f6699dSAlexei Starovoitov if (type != BPF_READ) 1237c4f6699dSAlexei Starovoitov return false; 1238c4f6699dSAlexei Starovoitov if (off % size != 0) 1239c4f6699dSAlexei Starovoitov return false; 124038207291SMartin KaFai Lau return true; 1241f1b9509cSAlexei Starovoitov } 1242f1b9509cSAlexei Starovoitov 1243f1b9509cSAlexei Starovoitov static bool tracing_prog_is_valid_access(int off, int size, 1244f1b9509cSAlexei Starovoitov enum bpf_access_type type, 1245f1b9509cSAlexei Starovoitov const struct bpf_prog *prog, 1246f1b9509cSAlexei Starovoitov struct bpf_insn_access_aux *info) 1247f1b9509cSAlexei Starovoitov { 1248f1b9509cSAlexei Starovoitov if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 1249f1b9509cSAlexei Starovoitov return false; 1250f1b9509cSAlexei Starovoitov if (type != BPF_READ) 1251f1b9509cSAlexei Starovoitov return false; 1252f1b9509cSAlexei Starovoitov if (off % size != 0) 1253f1b9509cSAlexei Starovoitov return false; 12549e15db66SAlexei Starovoitov return btf_ctx_access(off, size, type, prog, info); 1255c4f6699dSAlexei Starovoitov } 1256c4f6699dSAlexei Starovoitov 1257c4f6699dSAlexei Starovoitov const struct bpf_verifier_ops raw_tracepoint_verifier_ops = { 1258c4f6699dSAlexei Starovoitov .get_func_proto = raw_tp_prog_func_proto, 1259c4f6699dSAlexei Starovoitov .is_valid_access = raw_tp_prog_is_valid_access, 1260c4f6699dSAlexei Starovoitov }; 1261c4f6699dSAlexei Starovoitov 1262c4f6699dSAlexei Starovoitov const struct bpf_prog_ops raw_tracepoint_prog_ops = { 1263c4f6699dSAlexei Starovoitov }; 1264c4f6699dSAlexei Starovoitov 1265f1b9509cSAlexei Starovoitov const struct bpf_verifier_ops tracing_verifier_ops = { 1266f1b9509cSAlexei Starovoitov .get_func_proto = tracing_prog_func_proto, 1267f1b9509cSAlexei Starovoitov .is_valid_access = tracing_prog_is_valid_access, 1268f1b9509cSAlexei Starovoitov }; 1269f1b9509cSAlexei Starovoitov 1270f1b9509cSAlexei Starovoitov const struct bpf_prog_ops tracing_prog_ops = { 1271f1b9509cSAlexei Starovoitov }; 1272f1b9509cSAlexei Starovoitov 12739df1c28bSMatt Mullins static bool raw_tp_writable_prog_is_valid_access(int off, int size, 12749df1c28bSMatt Mullins enum bpf_access_type type, 12759df1c28bSMatt Mullins const struct bpf_prog *prog, 12769df1c28bSMatt Mullins struct bpf_insn_access_aux *info) 12779df1c28bSMatt Mullins { 12789df1c28bSMatt Mullins if (off == 0) { 12799df1c28bSMatt Mullins if (size != sizeof(u64) || type != BPF_READ) 12809df1c28bSMatt Mullins return false; 12819df1c28bSMatt Mullins info->reg_type = PTR_TO_TP_BUFFER; 12829df1c28bSMatt Mullins } 12839df1c28bSMatt Mullins return raw_tp_prog_is_valid_access(off, size, type, prog, info); 12849df1c28bSMatt Mullins } 12859df1c28bSMatt Mullins 12869df1c28bSMatt Mullins const struct bpf_verifier_ops raw_tracepoint_writable_verifier_ops = { 12879df1c28bSMatt Mullins .get_func_proto = raw_tp_prog_func_proto, 12889df1c28bSMatt Mullins .is_valid_access = raw_tp_writable_prog_is_valid_access, 12899df1c28bSMatt Mullins }; 12909df1c28bSMatt Mullins 12919df1c28bSMatt Mullins const struct bpf_prog_ops raw_tracepoint_writable_prog_ops = { 12929df1c28bSMatt Mullins }; 12939df1c28bSMatt Mullins 12940515e599SAlexei Starovoitov static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type, 12955e43f899SAndrey Ignatov const struct bpf_prog *prog, 129623994631SYonghong Song struct bpf_insn_access_aux *info) 12970515e599SAlexei Starovoitov { 129895da0cdbSTeng Qin const int size_u64 = sizeof(u64); 129931fd8581SYonghong Song 13000515e599SAlexei Starovoitov if (off < 0 || off >= sizeof(struct bpf_perf_event_data)) 13010515e599SAlexei Starovoitov return false; 13020515e599SAlexei Starovoitov if (type != BPF_READ) 13030515e599SAlexei Starovoitov return false; 1304bc23105cSDaniel Borkmann if (off % size != 0) { 1305bc23105cSDaniel Borkmann if (sizeof(unsigned long) != 4) 13060515e599SAlexei Starovoitov return false; 1307bc23105cSDaniel Borkmann if (size != 8) 1308bc23105cSDaniel Borkmann return false; 1309bc23105cSDaniel Borkmann if (off % size != 4) 1310bc23105cSDaniel Borkmann return false; 1311bc23105cSDaniel Borkmann } 131231fd8581SYonghong Song 1313f96da094SDaniel Borkmann switch (off) { 1314f96da094SDaniel Borkmann case bpf_ctx_range(struct bpf_perf_event_data, sample_period): 131595da0cdbSTeng Qin bpf_ctx_record_field_size(info, size_u64); 131695da0cdbSTeng Qin if (!bpf_ctx_narrow_access_ok(off, size, size_u64)) 131795da0cdbSTeng Qin return false; 131895da0cdbSTeng Qin break; 131995da0cdbSTeng Qin case bpf_ctx_range(struct bpf_perf_event_data, addr): 132095da0cdbSTeng Qin bpf_ctx_record_field_size(info, size_u64); 132195da0cdbSTeng Qin if (!bpf_ctx_narrow_access_ok(off, size, size_u64)) 132223994631SYonghong Song return false; 1323f96da094SDaniel Borkmann break; 1324f96da094SDaniel Borkmann default: 13250515e599SAlexei Starovoitov if (size != sizeof(long)) 13260515e599SAlexei Starovoitov return false; 13270515e599SAlexei Starovoitov } 1328f96da094SDaniel Borkmann 13290515e599SAlexei Starovoitov return true; 13300515e599SAlexei Starovoitov } 13310515e599SAlexei Starovoitov 13326b8cc1d1SDaniel Borkmann static u32 pe_prog_convert_ctx_access(enum bpf_access_type type, 13336b8cc1d1SDaniel Borkmann const struct bpf_insn *si, 13340515e599SAlexei Starovoitov struct bpf_insn *insn_buf, 1335f96da094SDaniel Borkmann struct bpf_prog *prog, u32 *target_size) 13360515e599SAlexei Starovoitov { 13370515e599SAlexei Starovoitov struct bpf_insn *insn = insn_buf; 13380515e599SAlexei Starovoitov 13396b8cc1d1SDaniel Borkmann switch (si->off) { 13400515e599SAlexei Starovoitov case offsetof(struct bpf_perf_event_data, sample_period): 1341f035a515SDaniel Borkmann *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 13426b8cc1d1SDaniel Borkmann data), si->dst_reg, si->src_reg, 13430515e599SAlexei Starovoitov offsetof(struct bpf_perf_event_data_kern, data)); 13446b8cc1d1SDaniel Borkmann *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, 1345f96da094SDaniel Borkmann bpf_target_off(struct perf_sample_data, period, 8, 1346f96da094SDaniel Borkmann target_size)); 13470515e599SAlexei Starovoitov break; 134895da0cdbSTeng Qin case offsetof(struct bpf_perf_event_data, addr): 134995da0cdbSTeng Qin *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 135095da0cdbSTeng Qin data), si->dst_reg, si->src_reg, 135195da0cdbSTeng Qin offsetof(struct bpf_perf_event_data_kern, data)); 135295da0cdbSTeng Qin *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, 135395da0cdbSTeng Qin bpf_target_off(struct perf_sample_data, addr, 8, 135495da0cdbSTeng Qin target_size)); 135595da0cdbSTeng Qin break; 13560515e599SAlexei Starovoitov default: 1357f035a515SDaniel Borkmann *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 13586b8cc1d1SDaniel Borkmann regs), si->dst_reg, si->src_reg, 13590515e599SAlexei Starovoitov offsetof(struct bpf_perf_event_data_kern, regs)); 13606b8cc1d1SDaniel Borkmann *insn++ = BPF_LDX_MEM(BPF_SIZEOF(long), si->dst_reg, si->dst_reg, 13616b8cc1d1SDaniel Borkmann si->off); 13620515e599SAlexei Starovoitov break; 13630515e599SAlexei Starovoitov } 13640515e599SAlexei Starovoitov 13650515e599SAlexei Starovoitov return insn - insn_buf; 13660515e599SAlexei Starovoitov } 13670515e599SAlexei Starovoitov 13687de16e3aSJakub Kicinski const struct bpf_verifier_ops perf_event_verifier_ops = { 1369f005afedSYonghong Song .get_func_proto = pe_prog_func_proto, 13700515e599SAlexei Starovoitov .is_valid_access = pe_prog_is_valid_access, 13710515e599SAlexei Starovoitov .convert_ctx_access = pe_prog_convert_ctx_access, 13720515e599SAlexei Starovoitov }; 13737de16e3aSJakub Kicinski 13747de16e3aSJakub Kicinski const struct bpf_prog_ops perf_event_prog_ops = { 13757de16e3aSJakub Kicinski }; 1376e87c6bc3SYonghong Song 1377e87c6bc3SYonghong Song static DEFINE_MUTEX(bpf_event_mutex); 1378e87c6bc3SYonghong Song 1379c8c088baSYonghong Song #define BPF_TRACE_MAX_PROGS 64 1380c8c088baSYonghong Song 1381e87c6bc3SYonghong Song int perf_event_attach_bpf_prog(struct perf_event *event, 1382e87c6bc3SYonghong Song struct bpf_prog *prog) 1383e87c6bc3SYonghong Song { 1384e672db03SStanislav Fomichev struct bpf_prog_array *old_array; 1385e87c6bc3SYonghong Song struct bpf_prog_array *new_array; 1386e87c6bc3SYonghong Song int ret = -EEXIST; 1387e87c6bc3SYonghong Song 13889802d865SJosef Bacik /* 1389b4da3340SMasami Hiramatsu * Kprobe override only works if they are on the function entry, 1390b4da3340SMasami Hiramatsu * and only if they are on the opt-in list. 13919802d865SJosef Bacik */ 13929802d865SJosef Bacik if (prog->kprobe_override && 1393b4da3340SMasami Hiramatsu (!trace_kprobe_on_func_entry(event->tp_event) || 13949802d865SJosef Bacik !trace_kprobe_error_injectable(event->tp_event))) 13959802d865SJosef Bacik return -EINVAL; 13969802d865SJosef Bacik 1397e87c6bc3SYonghong Song mutex_lock(&bpf_event_mutex); 1398e87c6bc3SYonghong Song 1399e87c6bc3SYonghong Song if (event->prog) 140007c41a29SYonghong Song goto unlock; 1401e87c6bc3SYonghong Song 1402e672db03SStanislav Fomichev old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); 1403c8c088baSYonghong Song if (old_array && 1404c8c088baSYonghong Song bpf_prog_array_length(old_array) >= BPF_TRACE_MAX_PROGS) { 1405c8c088baSYonghong Song ret = -E2BIG; 1406c8c088baSYonghong Song goto unlock; 1407c8c088baSYonghong Song } 1408c8c088baSYonghong Song 1409e87c6bc3SYonghong Song ret = bpf_prog_array_copy(old_array, NULL, prog, &new_array); 1410e87c6bc3SYonghong Song if (ret < 0) 141107c41a29SYonghong Song goto unlock; 1412e87c6bc3SYonghong Song 1413e87c6bc3SYonghong Song /* set the new array to event->tp_event and set event->prog */ 1414e87c6bc3SYonghong Song event->prog = prog; 1415e87c6bc3SYonghong Song rcu_assign_pointer(event->tp_event->prog_array, new_array); 1416e87c6bc3SYonghong Song bpf_prog_array_free(old_array); 1417e87c6bc3SYonghong Song 141807c41a29SYonghong Song unlock: 1419e87c6bc3SYonghong Song mutex_unlock(&bpf_event_mutex); 1420e87c6bc3SYonghong Song return ret; 1421e87c6bc3SYonghong Song } 1422e87c6bc3SYonghong Song 1423e87c6bc3SYonghong Song void perf_event_detach_bpf_prog(struct perf_event *event) 1424e87c6bc3SYonghong Song { 1425e672db03SStanislav Fomichev struct bpf_prog_array *old_array; 1426e87c6bc3SYonghong Song struct bpf_prog_array *new_array; 1427e87c6bc3SYonghong Song int ret; 1428e87c6bc3SYonghong Song 1429e87c6bc3SYonghong Song mutex_lock(&bpf_event_mutex); 1430e87c6bc3SYonghong Song 1431e87c6bc3SYonghong Song if (!event->prog) 143207c41a29SYonghong Song goto unlock; 1433e87c6bc3SYonghong Song 1434e672db03SStanislav Fomichev old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); 1435e87c6bc3SYonghong Song ret = bpf_prog_array_copy(old_array, event->prog, NULL, &new_array); 1436170a7e3eSSean Young if (ret == -ENOENT) 1437170a7e3eSSean Young goto unlock; 1438e87c6bc3SYonghong Song if (ret < 0) { 1439e87c6bc3SYonghong Song bpf_prog_array_delete_safe(old_array, event->prog); 1440e87c6bc3SYonghong Song } else { 1441e87c6bc3SYonghong Song rcu_assign_pointer(event->tp_event->prog_array, new_array); 1442e87c6bc3SYonghong Song bpf_prog_array_free(old_array); 1443e87c6bc3SYonghong Song } 1444e87c6bc3SYonghong Song 1445e87c6bc3SYonghong Song bpf_prog_put(event->prog); 1446e87c6bc3SYonghong Song event->prog = NULL; 1447e87c6bc3SYonghong Song 144807c41a29SYonghong Song unlock: 1449e87c6bc3SYonghong Song mutex_unlock(&bpf_event_mutex); 1450e87c6bc3SYonghong Song } 1451f371b304SYonghong Song 1452f4e2298eSYonghong Song int perf_event_query_prog_array(struct perf_event *event, void __user *info) 1453f371b304SYonghong Song { 1454f371b304SYonghong Song struct perf_event_query_bpf __user *uquery = info; 1455f371b304SYonghong Song struct perf_event_query_bpf query = {}; 1456e672db03SStanislav Fomichev struct bpf_prog_array *progs; 14573a38bb98SYonghong Song u32 *ids, prog_cnt, ids_len; 1458f371b304SYonghong Song int ret; 1459f371b304SYonghong Song 1460f371b304SYonghong Song if (!capable(CAP_SYS_ADMIN)) 1461f371b304SYonghong Song return -EPERM; 1462f371b304SYonghong Song if (event->attr.type != PERF_TYPE_TRACEPOINT) 1463f371b304SYonghong Song return -EINVAL; 1464f371b304SYonghong Song if (copy_from_user(&query, uquery, sizeof(query))) 1465f371b304SYonghong Song return -EFAULT; 14663a38bb98SYonghong Song 14673a38bb98SYonghong Song ids_len = query.ids_len; 14683a38bb98SYonghong Song if (ids_len > BPF_TRACE_MAX_PROGS) 14699c481b90SDaniel Borkmann return -E2BIG; 14703a38bb98SYonghong Song ids = kcalloc(ids_len, sizeof(u32), GFP_USER | __GFP_NOWARN); 14713a38bb98SYonghong Song if (!ids) 14723a38bb98SYonghong Song return -ENOMEM; 14733a38bb98SYonghong Song /* 14743a38bb98SYonghong Song * The above kcalloc returns ZERO_SIZE_PTR when ids_len = 0, which 14753a38bb98SYonghong Song * is required when user only wants to check for uquery->prog_cnt. 14763a38bb98SYonghong Song * There is no need to check for it since the case is handled 14773a38bb98SYonghong Song * gracefully in bpf_prog_array_copy_info. 14783a38bb98SYonghong Song */ 1479f371b304SYonghong Song 1480f371b304SYonghong Song mutex_lock(&bpf_event_mutex); 1481e672db03SStanislav Fomichev progs = bpf_event_rcu_dereference(event->tp_event->prog_array); 1482e672db03SStanislav Fomichev ret = bpf_prog_array_copy_info(progs, ids, ids_len, &prog_cnt); 1483f371b304SYonghong Song mutex_unlock(&bpf_event_mutex); 1484f371b304SYonghong Song 14853a38bb98SYonghong Song if (copy_to_user(&uquery->prog_cnt, &prog_cnt, sizeof(prog_cnt)) || 14863a38bb98SYonghong Song copy_to_user(uquery->ids, ids, ids_len * sizeof(u32))) 14873a38bb98SYonghong Song ret = -EFAULT; 14883a38bb98SYonghong Song 14893a38bb98SYonghong Song kfree(ids); 1490f371b304SYonghong Song return ret; 1491f371b304SYonghong Song } 1492c4f6699dSAlexei Starovoitov 1493c4f6699dSAlexei Starovoitov extern struct bpf_raw_event_map __start__bpf_raw_tp[]; 1494c4f6699dSAlexei Starovoitov extern struct bpf_raw_event_map __stop__bpf_raw_tp[]; 1495c4f6699dSAlexei Starovoitov 1496a38d1107SMatt Mullins struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name) 1497c4f6699dSAlexei Starovoitov { 1498c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp = __start__bpf_raw_tp; 1499c4f6699dSAlexei Starovoitov 1500c4f6699dSAlexei Starovoitov for (; btp < __stop__bpf_raw_tp; btp++) { 1501c4f6699dSAlexei Starovoitov if (!strcmp(btp->tp->name, name)) 1502c4f6699dSAlexei Starovoitov return btp; 1503c4f6699dSAlexei Starovoitov } 1504a38d1107SMatt Mullins 1505a38d1107SMatt Mullins return bpf_get_raw_tracepoint_module(name); 1506a38d1107SMatt Mullins } 1507a38d1107SMatt Mullins 1508a38d1107SMatt Mullins void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp) 1509a38d1107SMatt Mullins { 1510a38d1107SMatt Mullins struct module *mod = __module_address((unsigned long)btp); 1511a38d1107SMatt Mullins 1512a38d1107SMatt Mullins if (mod) 1513a38d1107SMatt Mullins module_put(mod); 1514c4f6699dSAlexei Starovoitov } 1515c4f6699dSAlexei Starovoitov 1516c4f6699dSAlexei Starovoitov static __always_inline 1517c4f6699dSAlexei Starovoitov void __bpf_trace_run(struct bpf_prog *prog, u64 *args) 1518c4f6699dSAlexei Starovoitov { 1519c4f6699dSAlexei Starovoitov rcu_read_lock(); 1520c4f6699dSAlexei Starovoitov preempt_disable(); 1521c4f6699dSAlexei Starovoitov (void) BPF_PROG_RUN(prog, args); 1522c4f6699dSAlexei Starovoitov preempt_enable(); 1523c4f6699dSAlexei Starovoitov rcu_read_unlock(); 1524c4f6699dSAlexei Starovoitov } 1525c4f6699dSAlexei Starovoitov 1526c4f6699dSAlexei Starovoitov #define UNPACK(...) __VA_ARGS__ 1527c4f6699dSAlexei Starovoitov #define REPEAT_1(FN, DL, X, ...) FN(X) 1528c4f6699dSAlexei Starovoitov #define REPEAT_2(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_1(FN, DL, __VA_ARGS__) 1529c4f6699dSAlexei Starovoitov #define REPEAT_3(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_2(FN, DL, __VA_ARGS__) 1530c4f6699dSAlexei Starovoitov #define REPEAT_4(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_3(FN, DL, __VA_ARGS__) 1531c4f6699dSAlexei Starovoitov #define REPEAT_5(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_4(FN, DL, __VA_ARGS__) 1532c4f6699dSAlexei Starovoitov #define REPEAT_6(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_5(FN, DL, __VA_ARGS__) 1533c4f6699dSAlexei Starovoitov #define REPEAT_7(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_6(FN, DL, __VA_ARGS__) 1534c4f6699dSAlexei Starovoitov #define REPEAT_8(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_7(FN, DL, __VA_ARGS__) 1535c4f6699dSAlexei Starovoitov #define REPEAT_9(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_8(FN, DL, __VA_ARGS__) 1536c4f6699dSAlexei Starovoitov #define REPEAT_10(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_9(FN, DL, __VA_ARGS__) 1537c4f6699dSAlexei Starovoitov #define REPEAT_11(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_10(FN, DL, __VA_ARGS__) 1538c4f6699dSAlexei Starovoitov #define REPEAT_12(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_11(FN, DL, __VA_ARGS__) 1539c4f6699dSAlexei Starovoitov #define REPEAT(X, FN, DL, ...) REPEAT_##X(FN, DL, __VA_ARGS__) 1540c4f6699dSAlexei Starovoitov 1541c4f6699dSAlexei Starovoitov #define SARG(X) u64 arg##X 1542c4f6699dSAlexei Starovoitov #define COPY(X) args[X] = arg##X 1543c4f6699dSAlexei Starovoitov 1544c4f6699dSAlexei Starovoitov #define __DL_COM (,) 1545c4f6699dSAlexei Starovoitov #define __DL_SEM (;) 1546c4f6699dSAlexei Starovoitov 1547c4f6699dSAlexei Starovoitov #define __SEQ_0_11 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 1548c4f6699dSAlexei Starovoitov 1549c4f6699dSAlexei Starovoitov #define BPF_TRACE_DEFN_x(x) \ 1550c4f6699dSAlexei Starovoitov void bpf_trace_run##x(struct bpf_prog *prog, \ 1551c4f6699dSAlexei Starovoitov REPEAT(x, SARG, __DL_COM, __SEQ_0_11)) \ 1552c4f6699dSAlexei Starovoitov { \ 1553c4f6699dSAlexei Starovoitov u64 args[x]; \ 1554c4f6699dSAlexei Starovoitov REPEAT(x, COPY, __DL_SEM, __SEQ_0_11); \ 1555c4f6699dSAlexei Starovoitov __bpf_trace_run(prog, args); \ 1556c4f6699dSAlexei Starovoitov } \ 1557c4f6699dSAlexei Starovoitov EXPORT_SYMBOL_GPL(bpf_trace_run##x) 1558c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(1); 1559c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(2); 1560c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(3); 1561c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(4); 1562c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(5); 1563c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(6); 1564c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(7); 1565c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(8); 1566c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(9); 1567c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(10); 1568c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(11); 1569c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(12); 1570c4f6699dSAlexei Starovoitov 1571c4f6699dSAlexei Starovoitov static int __bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog) 1572c4f6699dSAlexei Starovoitov { 1573c4f6699dSAlexei Starovoitov struct tracepoint *tp = btp->tp; 1574c4f6699dSAlexei Starovoitov 1575c4f6699dSAlexei Starovoitov /* 1576c4f6699dSAlexei Starovoitov * check that program doesn't access arguments beyond what's 1577c4f6699dSAlexei Starovoitov * available in this tracepoint 1578c4f6699dSAlexei Starovoitov */ 1579c4f6699dSAlexei Starovoitov if (prog->aux->max_ctx_offset > btp->num_args * sizeof(u64)) 1580c4f6699dSAlexei Starovoitov return -EINVAL; 1581c4f6699dSAlexei Starovoitov 15829df1c28bSMatt Mullins if (prog->aux->max_tp_access > btp->writable_size) 15839df1c28bSMatt Mullins return -EINVAL; 15849df1c28bSMatt Mullins 1585c4f6699dSAlexei Starovoitov return tracepoint_probe_register(tp, (void *)btp->bpf_func, prog); 1586c4f6699dSAlexei Starovoitov } 1587c4f6699dSAlexei Starovoitov 1588c4f6699dSAlexei Starovoitov int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog) 1589c4f6699dSAlexei Starovoitov { 1590e16ec340SAlexei Starovoitov return __bpf_probe_register(btp, prog); 1591c4f6699dSAlexei Starovoitov } 1592c4f6699dSAlexei Starovoitov 1593c4f6699dSAlexei Starovoitov int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog) 1594c4f6699dSAlexei Starovoitov { 1595e16ec340SAlexei Starovoitov return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, prog); 1596c4f6699dSAlexei Starovoitov } 159741bdc4b4SYonghong Song 159841bdc4b4SYonghong Song int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, 159941bdc4b4SYonghong Song u32 *fd_type, const char **buf, 160041bdc4b4SYonghong Song u64 *probe_offset, u64 *probe_addr) 160141bdc4b4SYonghong Song { 160241bdc4b4SYonghong Song bool is_tracepoint, is_syscall_tp; 160341bdc4b4SYonghong Song struct bpf_prog *prog; 160441bdc4b4SYonghong Song int flags, err = 0; 160541bdc4b4SYonghong Song 160641bdc4b4SYonghong Song prog = event->prog; 160741bdc4b4SYonghong Song if (!prog) 160841bdc4b4SYonghong Song return -ENOENT; 160941bdc4b4SYonghong Song 161041bdc4b4SYonghong Song /* not supporting BPF_PROG_TYPE_PERF_EVENT yet */ 161141bdc4b4SYonghong Song if (prog->type == BPF_PROG_TYPE_PERF_EVENT) 161241bdc4b4SYonghong Song return -EOPNOTSUPP; 161341bdc4b4SYonghong Song 161441bdc4b4SYonghong Song *prog_id = prog->aux->id; 161541bdc4b4SYonghong Song flags = event->tp_event->flags; 161641bdc4b4SYonghong Song is_tracepoint = flags & TRACE_EVENT_FL_TRACEPOINT; 161741bdc4b4SYonghong Song is_syscall_tp = is_syscall_trace_event(event->tp_event); 161841bdc4b4SYonghong Song 161941bdc4b4SYonghong Song if (is_tracepoint || is_syscall_tp) { 162041bdc4b4SYonghong Song *buf = is_tracepoint ? event->tp_event->tp->name 162141bdc4b4SYonghong Song : event->tp_event->name; 162241bdc4b4SYonghong Song *fd_type = BPF_FD_TYPE_TRACEPOINT; 162341bdc4b4SYonghong Song *probe_offset = 0x0; 162441bdc4b4SYonghong Song *probe_addr = 0x0; 162541bdc4b4SYonghong Song } else { 162641bdc4b4SYonghong Song /* kprobe/uprobe */ 162741bdc4b4SYonghong Song err = -EOPNOTSUPP; 162841bdc4b4SYonghong Song #ifdef CONFIG_KPROBE_EVENTS 162941bdc4b4SYonghong Song if (flags & TRACE_EVENT_FL_KPROBE) 163041bdc4b4SYonghong Song err = bpf_get_kprobe_info(event, fd_type, buf, 163141bdc4b4SYonghong Song probe_offset, probe_addr, 163241bdc4b4SYonghong Song event->attr.type == PERF_TYPE_TRACEPOINT); 163341bdc4b4SYonghong Song #endif 163441bdc4b4SYonghong Song #ifdef CONFIG_UPROBE_EVENTS 163541bdc4b4SYonghong Song if (flags & TRACE_EVENT_FL_UPROBE) 163641bdc4b4SYonghong Song err = bpf_get_uprobe_info(event, fd_type, buf, 163741bdc4b4SYonghong Song probe_offset, 163841bdc4b4SYonghong Song event->attr.type == PERF_TYPE_TRACEPOINT); 163941bdc4b4SYonghong Song #endif 164041bdc4b4SYonghong Song } 164141bdc4b4SYonghong Song 164241bdc4b4SYonghong Song return err; 164341bdc4b4SYonghong Song } 1644a38d1107SMatt Mullins 16459db1ff0aSYonghong Song static int __init send_signal_irq_work_init(void) 16469db1ff0aSYonghong Song { 16479db1ff0aSYonghong Song int cpu; 16489db1ff0aSYonghong Song struct send_signal_irq_work *work; 16499db1ff0aSYonghong Song 16509db1ff0aSYonghong Song for_each_possible_cpu(cpu) { 16519db1ff0aSYonghong Song work = per_cpu_ptr(&send_signal_work, cpu); 16529db1ff0aSYonghong Song init_irq_work(&work->irq_work, do_bpf_send_signal); 16539db1ff0aSYonghong Song } 16549db1ff0aSYonghong Song return 0; 16559db1ff0aSYonghong Song } 16569db1ff0aSYonghong Song 16579db1ff0aSYonghong Song subsys_initcall(send_signal_irq_work_init); 16589db1ff0aSYonghong Song 1659a38d1107SMatt Mullins #ifdef CONFIG_MODULES 1660390e99cfSStanislav Fomichev static int bpf_event_notify(struct notifier_block *nb, unsigned long op, 1661390e99cfSStanislav Fomichev void *module) 1662a38d1107SMatt Mullins { 1663a38d1107SMatt Mullins struct bpf_trace_module *btm, *tmp; 1664a38d1107SMatt Mullins struct module *mod = module; 1665a38d1107SMatt Mullins 1666a38d1107SMatt Mullins if (mod->num_bpf_raw_events == 0 || 1667a38d1107SMatt Mullins (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING)) 1668a38d1107SMatt Mullins return 0; 1669a38d1107SMatt Mullins 1670a38d1107SMatt Mullins mutex_lock(&bpf_module_mutex); 1671a38d1107SMatt Mullins 1672a38d1107SMatt Mullins switch (op) { 1673a38d1107SMatt Mullins case MODULE_STATE_COMING: 1674a38d1107SMatt Mullins btm = kzalloc(sizeof(*btm), GFP_KERNEL); 1675a38d1107SMatt Mullins if (btm) { 1676a38d1107SMatt Mullins btm->module = module; 1677a38d1107SMatt Mullins list_add(&btm->list, &bpf_trace_modules); 1678a38d1107SMatt Mullins } 1679a38d1107SMatt Mullins break; 1680a38d1107SMatt Mullins case MODULE_STATE_GOING: 1681a38d1107SMatt Mullins list_for_each_entry_safe(btm, tmp, &bpf_trace_modules, list) { 1682a38d1107SMatt Mullins if (btm->module == module) { 1683a38d1107SMatt Mullins list_del(&btm->list); 1684a38d1107SMatt Mullins kfree(btm); 1685a38d1107SMatt Mullins break; 1686a38d1107SMatt Mullins } 1687a38d1107SMatt Mullins } 1688a38d1107SMatt Mullins break; 1689a38d1107SMatt Mullins } 1690a38d1107SMatt Mullins 1691a38d1107SMatt Mullins mutex_unlock(&bpf_module_mutex); 1692a38d1107SMatt Mullins 1693a38d1107SMatt Mullins return 0; 1694a38d1107SMatt Mullins } 1695a38d1107SMatt Mullins 1696a38d1107SMatt Mullins static struct notifier_block bpf_module_nb = { 1697a38d1107SMatt Mullins .notifier_call = bpf_event_notify, 1698a38d1107SMatt Mullins }; 1699a38d1107SMatt Mullins 1700390e99cfSStanislav Fomichev static int __init bpf_event_init(void) 1701a38d1107SMatt Mullins { 1702a38d1107SMatt Mullins register_module_notifier(&bpf_module_nb); 1703a38d1107SMatt Mullins return 0; 1704a38d1107SMatt Mullins } 1705a38d1107SMatt Mullins 1706a38d1107SMatt Mullins fs_initcall(bpf_event_init); 1707a38d1107SMatt Mullins #endif /* CONFIG_MODULES */ 1708