1179a0cc4SSteven Rostedt (VMware) // SPDX-License-Identifier: GPL-2.0 22541517cSAlexei Starovoitov /* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com 30515e599SAlexei Starovoitov * Copyright (c) 2016 Facebook 42541517cSAlexei Starovoitov */ 52541517cSAlexei Starovoitov #include <linux/kernel.h> 62541517cSAlexei Starovoitov #include <linux/types.h> 72541517cSAlexei Starovoitov #include <linux/slab.h> 82541517cSAlexei Starovoitov #include <linux/bpf.h> 90515e599SAlexei Starovoitov #include <linux/bpf_perf_event.h> 10c4d0bfb4SAlan Maguire #include <linux/btf.h> 112541517cSAlexei Starovoitov #include <linux/filter.h> 122541517cSAlexei Starovoitov #include <linux/uaccess.h> 139c959c86SAlexei Starovoitov #include <linux/ctype.h> 149802d865SJosef Bacik #include <linux/kprobes.h> 15ac5a72eaSAlan Maguire #include <linux/spinlock.h> 1641bdc4b4SYonghong Song #include <linux/syscalls.h> 17540adea3SMasami Hiramatsu #include <linux/error-injection.h> 18c9a0f3b8SJiri Olsa #include <linux/btf_ids.h> 196f100640SKP Singh #include <linux/bpf_lsm.h> 206f100640SKP Singh 218e4597c6SMartin KaFai Lau #include <net/bpf_sk_storage.h> 229802d865SJosef Bacik 23c4d0bfb4SAlan Maguire #include <uapi/linux/bpf.h> 24c4d0bfb4SAlan Maguire #include <uapi/linux/btf.h> 25c4d0bfb4SAlan Maguire 26c7b6f29bSNadav Amit #include <asm/tlb.h> 27c7b6f29bSNadav Amit 289802d865SJosef Bacik #include "trace_probe.h" 292541517cSAlexei Starovoitov #include "trace.h" 302541517cSAlexei Starovoitov 31ac5a72eaSAlan Maguire #define CREATE_TRACE_POINTS 32ac5a72eaSAlan Maguire #include "bpf_trace.h" 33ac5a72eaSAlan Maguire 34e672db03SStanislav Fomichev #define bpf_event_rcu_dereference(p) \ 35e672db03SStanislav Fomichev rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex)) 36e672db03SStanislav Fomichev 37a38d1107SMatt Mullins #ifdef CONFIG_MODULES 38a38d1107SMatt Mullins struct bpf_trace_module { 39a38d1107SMatt Mullins struct module *module; 40a38d1107SMatt Mullins struct list_head list; 41a38d1107SMatt Mullins }; 42a38d1107SMatt Mullins 43a38d1107SMatt Mullins static LIST_HEAD(bpf_trace_modules); 44a38d1107SMatt Mullins static DEFINE_MUTEX(bpf_module_mutex); 45a38d1107SMatt Mullins 46a38d1107SMatt Mullins static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name) 47a38d1107SMatt Mullins { 48a38d1107SMatt Mullins struct bpf_raw_event_map *btp, *ret = NULL; 49a38d1107SMatt Mullins struct bpf_trace_module *btm; 50a38d1107SMatt Mullins unsigned int i; 51a38d1107SMatt Mullins 52a38d1107SMatt Mullins mutex_lock(&bpf_module_mutex); 53a38d1107SMatt Mullins list_for_each_entry(btm, &bpf_trace_modules, list) { 54a38d1107SMatt Mullins for (i = 0; i < btm->module->num_bpf_raw_events; ++i) { 55a38d1107SMatt Mullins btp = &btm->module->bpf_raw_events[i]; 56a38d1107SMatt Mullins if (!strcmp(btp->tp->name, name)) { 57a38d1107SMatt Mullins if (try_module_get(btm->module)) 58a38d1107SMatt Mullins ret = btp; 59a38d1107SMatt Mullins goto out; 60a38d1107SMatt Mullins } 61a38d1107SMatt Mullins } 62a38d1107SMatt Mullins } 63a38d1107SMatt Mullins out: 64a38d1107SMatt Mullins mutex_unlock(&bpf_module_mutex); 65a38d1107SMatt Mullins return ret; 66a38d1107SMatt Mullins } 67a38d1107SMatt Mullins #else 68a38d1107SMatt Mullins static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name) 69a38d1107SMatt Mullins { 70a38d1107SMatt Mullins return NULL; 71a38d1107SMatt Mullins } 72a38d1107SMatt Mullins #endif /* CONFIG_MODULES */ 73a38d1107SMatt Mullins 74035226b9SGianluca Borello u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 75c195651eSYonghong Song u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 76035226b9SGianluca Borello 77eb411377SAlan Maguire static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size, 78eb411377SAlan Maguire u64 flags, const struct btf **btf, 79eb411377SAlan Maguire s32 *btf_id); 80eb411377SAlan Maguire 812541517cSAlexei Starovoitov /** 822541517cSAlexei Starovoitov * trace_call_bpf - invoke BPF program 83e87c6bc3SYonghong Song * @call: tracepoint event 842541517cSAlexei Starovoitov * @ctx: opaque context pointer 852541517cSAlexei Starovoitov * 862541517cSAlexei Starovoitov * kprobe handlers execute BPF programs via this helper. 872541517cSAlexei Starovoitov * Can be used from static tracepoints in the future. 882541517cSAlexei Starovoitov * 892541517cSAlexei Starovoitov * Return: BPF programs always return an integer which is interpreted by 902541517cSAlexei Starovoitov * kprobe handler as: 912541517cSAlexei Starovoitov * 0 - return from kprobe (event is filtered out) 922541517cSAlexei Starovoitov * 1 - store kprobe event into ring buffer 932541517cSAlexei Starovoitov * Other values are reserved and currently alias to 1 942541517cSAlexei Starovoitov */ 95e87c6bc3SYonghong Song unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) 962541517cSAlexei Starovoitov { 972541517cSAlexei Starovoitov unsigned int ret; 982541517cSAlexei Starovoitov 992541517cSAlexei Starovoitov if (in_nmi()) /* not supported yet */ 1002541517cSAlexei Starovoitov return 1; 1012541517cSAlexei Starovoitov 102b0a81b94SThomas Gleixner cant_sleep(); 1032541517cSAlexei Starovoitov 1042541517cSAlexei Starovoitov if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { 1052541517cSAlexei Starovoitov /* 1062541517cSAlexei Starovoitov * since some bpf program is already running on this cpu, 1072541517cSAlexei Starovoitov * don't call into another bpf program (same or different) 1082541517cSAlexei Starovoitov * and don't send kprobe event into ring-buffer, 1092541517cSAlexei Starovoitov * so return zero here 1102541517cSAlexei Starovoitov */ 1112541517cSAlexei Starovoitov ret = 0; 1122541517cSAlexei Starovoitov goto out; 1132541517cSAlexei Starovoitov } 1142541517cSAlexei Starovoitov 115e87c6bc3SYonghong Song /* 116e87c6bc3SYonghong Song * Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock 117e87c6bc3SYonghong Song * to all call sites, we did a bpf_prog_array_valid() there to check 118e87c6bc3SYonghong Song * whether call->prog_array is empty or not, which is 119e87c6bc3SYonghong Song * a heurisitc to speed up execution. 120e87c6bc3SYonghong Song * 121e87c6bc3SYonghong Song * If bpf_prog_array_valid() fetched prog_array was 122e87c6bc3SYonghong Song * non-NULL, we go into trace_call_bpf() and do the actual 123e87c6bc3SYonghong Song * proper rcu_dereference() under RCU lock. 124e87c6bc3SYonghong Song * If it turns out that prog_array is NULL then, we bail out. 125e87c6bc3SYonghong Song * For the opposite, if the bpf_prog_array_valid() fetched pointer 126e87c6bc3SYonghong Song * was NULL, you'll skip the prog_array with the risk of missing 127e87c6bc3SYonghong Song * out of events when it was updated in between this and the 128e87c6bc3SYonghong Song * rcu_dereference() which is accepted risk. 129e87c6bc3SYonghong Song */ 130e87c6bc3SYonghong Song ret = BPF_PROG_RUN_ARRAY_CHECK(call->prog_array, ctx, BPF_PROG_RUN); 1312541517cSAlexei Starovoitov 1322541517cSAlexei Starovoitov out: 1332541517cSAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 1342541517cSAlexei Starovoitov 1352541517cSAlexei Starovoitov return ret; 1362541517cSAlexei Starovoitov } 1372541517cSAlexei Starovoitov 1389802d865SJosef Bacik #ifdef CONFIG_BPF_KPROBE_OVERRIDE 1399802d865SJosef Bacik BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc) 1409802d865SJosef Bacik { 1419802d865SJosef Bacik regs_set_return_value(regs, rc); 142540adea3SMasami Hiramatsu override_function_with_return(regs); 1439802d865SJosef Bacik return 0; 1449802d865SJosef Bacik } 1459802d865SJosef Bacik 1469802d865SJosef Bacik static const struct bpf_func_proto bpf_override_return_proto = { 1479802d865SJosef Bacik .func = bpf_override_return, 1489802d865SJosef Bacik .gpl_only = true, 1499802d865SJosef Bacik .ret_type = RET_INTEGER, 1509802d865SJosef Bacik .arg1_type = ARG_PTR_TO_CTX, 1519802d865SJosef Bacik .arg2_type = ARG_ANYTHING, 1529802d865SJosef Bacik }; 1539802d865SJosef Bacik #endif 1549802d865SJosef Bacik 1558d92db5cSChristoph Hellwig static __always_inline int 1568d92db5cSChristoph Hellwig bpf_probe_read_user_common(void *dst, u32 size, const void __user *unsafe_ptr) 1578d92db5cSChristoph Hellwig { 1588d92db5cSChristoph Hellwig int ret; 1598d92db5cSChristoph Hellwig 160c0ee37e8SChristoph Hellwig ret = copy_from_user_nofault(dst, unsafe_ptr, size); 1618d92db5cSChristoph Hellwig if (unlikely(ret < 0)) 1628d92db5cSChristoph Hellwig memset(dst, 0, size); 1638d92db5cSChristoph Hellwig return ret; 1648d92db5cSChristoph Hellwig } 1658d92db5cSChristoph Hellwig 1666ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_user, void *, dst, u32, size, 1676ae08ae3SDaniel Borkmann const void __user *, unsafe_ptr) 1682541517cSAlexei Starovoitov { 1698d92db5cSChristoph Hellwig return bpf_probe_read_user_common(dst, size, unsafe_ptr); 1702541517cSAlexei Starovoitov } 1712541517cSAlexei Starovoitov 172f470378cSJohn Fastabend const struct bpf_func_proto bpf_probe_read_user_proto = { 1736ae08ae3SDaniel Borkmann .func = bpf_probe_read_user, 1746ae08ae3SDaniel Borkmann .gpl_only = true, 1756ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 1766ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 1776ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 1786ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 1796ae08ae3SDaniel Borkmann }; 1806ae08ae3SDaniel Borkmann 1818d92db5cSChristoph Hellwig static __always_inline int 1828d92db5cSChristoph Hellwig bpf_probe_read_user_str_common(void *dst, u32 size, 1838d92db5cSChristoph Hellwig const void __user *unsafe_ptr) 1848d92db5cSChristoph Hellwig { 1858d92db5cSChristoph Hellwig int ret; 1868d92db5cSChristoph Hellwig 1878d92db5cSChristoph Hellwig ret = strncpy_from_user_nofault(dst, unsafe_ptr, size); 1888d92db5cSChristoph Hellwig if (unlikely(ret < 0)) 1898d92db5cSChristoph Hellwig memset(dst, 0, size); 1908d92db5cSChristoph Hellwig return ret; 1918d92db5cSChristoph Hellwig } 1928d92db5cSChristoph Hellwig 1936ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_user_str, void *, dst, u32, size, 1946ae08ae3SDaniel Borkmann const void __user *, unsafe_ptr) 1956ae08ae3SDaniel Borkmann { 1968d92db5cSChristoph Hellwig return bpf_probe_read_user_str_common(dst, size, unsafe_ptr); 1976ae08ae3SDaniel Borkmann } 1986ae08ae3SDaniel Borkmann 199f470378cSJohn Fastabend const struct bpf_func_proto bpf_probe_read_user_str_proto = { 2006ae08ae3SDaniel Borkmann .func = bpf_probe_read_user_str, 2016ae08ae3SDaniel Borkmann .gpl_only = true, 2026ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 2036ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 2046ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 2056ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 2066ae08ae3SDaniel Borkmann }; 2076ae08ae3SDaniel Borkmann 2086ae08ae3SDaniel Borkmann static __always_inline int 2098d92db5cSChristoph Hellwig bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr) 2106ae08ae3SDaniel Borkmann { 2116ae08ae3SDaniel Borkmann int ret = security_locked_down(LOCKDOWN_BPF_READ); 2126ae08ae3SDaniel Borkmann 2136ae08ae3SDaniel Borkmann if (unlikely(ret < 0)) 2148d92db5cSChristoph Hellwig goto fail; 215fe557319SChristoph Hellwig ret = copy_from_kernel_nofault(dst, unsafe_ptr, size); 2166ae08ae3SDaniel Borkmann if (unlikely(ret < 0)) 2178d92db5cSChristoph Hellwig goto fail; 2188d92db5cSChristoph Hellwig return ret; 2198d92db5cSChristoph Hellwig fail: 2206ae08ae3SDaniel Borkmann memset(dst, 0, size); 2216ae08ae3SDaniel Borkmann return ret; 2226ae08ae3SDaniel Borkmann } 2236ae08ae3SDaniel Borkmann 2246ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_kernel, void *, dst, u32, size, 2256ae08ae3SDaniel Borkmann const void *, unsafe_ptr) 2266ae08ae3SDaniel Borkmann { 2278d92db5cSChristoph Hellwig return bpf_probe_read_kernel_common(dst, size, unsafe_ptr); 2286ae08ae3SDaniel Borkmann } 2296ae08ae3SDaniel Borkmann 230f470378cSJohn Fastabend const struct bpf_func_proto bpf_probe_read_kernel_proto = { 2316ae08ae3SDaniel Borkmann .func = bpf_probe_read_kernel, 2326ae08ae3SDaniel Borkmann .gpl_only = true, 2336ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 2346ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 2356ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 2366ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 2376ae08ae3SDaniel Borkmann }; 2386ae08ae3SDaniel Borkmann 2396ae08ae3SDaniel Borkmann static __always_inline int 2408d92db5cSChristoph Hellwig bpf_probe_read_kernel_str_common(void *dst, u32 size, const void *unsafe_ptr) 2416ae08ae3SDaniel Borkmann { 2426ae08ae3SDaniel Borkmann int ret = security_locked_down(LOCKDOWN_BPF_READ); 2436ae08ae3SDaniel Borkmann 2446ae08ae3SDaniel Borkmann if (unlikely(ret < 0)) 2458d92db5cSChristoph Hellwig goto fail; 2468d92db5cSChristoph Hellwig 2476ae08ae3SDaniel Borkmann /* 2488d92db5cSChristoph Hellwig * The strncpy_from_kernel_nofault() call will likely not fill the 2498d92db5cSChristoph Hellwig * entire buffer, but that's okay in this circumstance as we're probing 2506ae08ae3SDaniel Borkmann * arbitrary memory anyway similar to bpf_probe_read_*() and might 2516ae08ae3SDaniel Borkmann * as well probe the stack. Thus, memory is explicitly cleared 2526ae08ae3SDaniel Borkmann * only in error case, so that improper users ignoring return 2536ae08ae3SDaniel Borkmann * code altogether don't copy garbage; otherwise length of string 2546ae08ae3SDaniel Borkmann * is returned that can be used for bpf_perf_event_output() et al. 2556ae08ae3SDaniel Borkmann */ 2568d92db5cSChristoph Hellwig ret = strncpy_from_kernel_nofault(dst, unsafe_ptr, size); 2576ae08ae3SDaniel Borkmann if (unlikely(ret < 0)) 2588d92db5cSChristoph Hellwig goto fail; 2598d92db5cSChristoph Hellwig 26002553b91SAndrii Nakryiko return ret; 2618d92db5cSChristoph Hellwig fail: 2626ae08ae3SDaniel Borkmann memset(dst, 0, size); 2636ae08ae3SDaniel Borkmann return ret; 2646ae08ae3SDaniel Borkmann } 2656ae08ae3SDaniel Borkmann 2666ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_kernel_str, void *, dst, u32, size, 2676ae08ae3SDaniel Borkmann const void *, unsafe_ptr) 2686ae08ae3SDaniel Borkmann { 2698d92db5cSChristoph Hellwig return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr); 2706ae08ae3SDaniel Borkmann } 2716ae08ae3SDaniel Borkmann 272f470378cSJohn Fastabend const struct bpf_func_proto bpf_probe_read_kernel_str_proto = { 2736ae08ae3SDaniel Borkmann .func = bpf_probe_read_kernel_str, 2746ae08ae3SDaniel Borkmann .gpl_only = true, 2756ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 2766ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 2776ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 2786ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 2796ae08ae3SDaniel Borkmann }; 2806ae08ae3SDaniel Borkmann 2818d92db5cSChristoph Hellwig #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE 2828d92db5cSChristoph Hellwig BPF_CALL_3(bpf_probe_read_compat, void *, dst, u32, size, 2838d92db5cSChristoph Hellwig const void *, unsafe_ptr) 2848d92db5cSChristoph Hellwig { 2858d92db5cSChristoph Hellwig if ((unsigned long)unsafe_ptr < TASK_SIZE) { 2868d92db5cSChristoph Hellwig return bpf_probe_read_user_common(dst, size, 2878d92db5cSChristoph Hellwig (__force void __user *)unsafe_ptr); 2888d92db5cSChristoph Hellwig } 2898d92db5cSChristoph Hellwig return bpf_probe_read_kernel_common(dst, size, unsafe_ptr); 2908d92db5cSChristoph Hellwig } 2918d92db5cSChristoph Hellwig 2928d92db5cSChristoph Hellwig static const struct bpf_func_proto bpf_probe_read_compat_proto = { 2938d92db5cSChristoph Hellwig .func = bpf_probe_read_compat, 2948d92db5cSChristoph Hellwig .gpl_only = true, 2958d92db5cSChristoph Hellwig .ret_type = RET_INTEGER, 2968d92db5cSChristoph Hellwig .arg1_type = ARG_PTR_TO_UNINIT_MEM, 2978d92db5cSChristoph Hellwig .arg2_type = ARG_CONST_SIZE_OR_ZERO, 2988d92db5cSChristoph Hellwig .arg3_type = ARG_ANYTHING, 2998d92db5cSChristoph Hellwig }; 3008d92db5cSChristoph Hellwig 3016ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_compat_str, void *, dst, u32, size, 3026ae08ae3SDaniel Borkmann const void *, unsafe_ptr) 3036ae08ae3SDaniel Borkmann { 3048d92db5cSChristoph Hellwig if ((unsigned long)unsafe_ptr < TASK_SIZE) { 3058d92db5cSChristoph Hellwig return bpf_probe_read_user_str_common(dst, size, 3068d92db5cSChristoph Hellwig (__force void __user *)unsafe_ptr); 3078d92db5cSChristoph Hellwig } 3088d92db5cSChristoph Hellwig return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr); 3096ae08ae3SDaniel Borkmann } 3106ae08ae3SDaniel Borkmann 3116ae08ae3SDaniel Borkmann static const struct bpf_func_proto bpf_probe_read_compat_str_proto = { 3126ae08ae3SDaniel Borkmann .func = bpf_probe_read_compat_str, 3132541517cSAlexei Starovoitov .gpl_only = true, 3142541517cSAlexei Starovoitov .ret_type = RET_INTEGER, 31539f19ebbSAlexei Starovoitov .arg1_type = ARG_PTR_TO_UNINIT_MEM, 3169c019e2bSYonghong Song .arg2_type = ARG_CONST_SIZE_OR_ZERO, 3172541517cSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 3182541517cSAlexei Starovoitov }; 3198d92db5cSChristoph Hellwig #endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */ 3202541517cSAlexei Starovoitov 321eb1b6688SDaniel Borkmann BPF_CALL_3(bpf_probe_write_user, void __user *, unsafe_ptr, const void *, src, 322f3694e00SDaniel Borkmann u32, size) 32396ae5227SSargun Dhillon { 32496ae5227SSargun Dhillon /* 32596ae5227SSargun Dhillon * Ensure we're in user context which is safe for the helper to 32696ae5227SSargun Dhillon * run. This helper has no business in a kthread. 32796ae5227SSargun Dhillon * 32896ae5227SSargun Dhillon * access_ok() should prevent writing to non-user memory, but in 32996ae5227SSargun Dhillon * some situations (nommu, temporary switch, etc) access_ok() does 33096ae5227SSargun Dhillon * not provide enough validation, hence the check on KERNEL_DS. 331c7b6f29bSNadav Amit * 332c7b6f29bSNadav Amit * nmi_uaccess_okay() ensures the probe is not run in an interim 333c7b6f29bSNadav Amit * state, when the task or mm are switched. This is specifically 334c7b6f29bSNadav Amit * required to prevent the use of temporary mm. 33596ae5227SSargun Dhillon */ 33696ae5227SSargun Dhillon 33796ae5227SSargun Dhillon if (unlikely(in_interrupt() || 33896ae5227SSargun Dhillon current->flags & (PF_KTHREAD | PF_EXITING))) 33996ae5227SSargun Dhillon return -EPERM; 340db68ce10SAl Viro if (unlikely(uaccess_kernel())) 34196ae5227SSargun Dhillon return -EPERM; 342c7b6f29bSNadav Amit if (unlikely(!nmi_uaccess_okay())) 343c7b6f29bSNadav Amit return -EPERM; 34496ae5227SSargun Dhillon 345c0ee37e8SChristoph Hellwig return copy_to_user_nofault(unsafe_ptr, src, size); 34696ae5227SSargun Dhillon } 34796ae5227SSargun Dhillon 34896ae5227SSargun Dhillon static const struct bpf_func_proto bpf_probe_write_user_proto = { 34996ae5227SSargun Dhillon .func = bpf_probe_write_user, 35096ae5227SSargun Dhillon .gpl_only = true, 35196ae5227SSargun Dhillon .ret_type = RET_INTEGER, 35296ae5227SSargun Dhillon .arg1_type = ARG_ANYTHING, 35339f19ebbSAlexei Starovoitov .arg2_type = ARG_PTR_TO_MEM, 35439f19ebbSAlexei Starovoitov .arg3_type = ARG_CONST_SIZE, 35596ae5227SSargun Dhillon }; 35696ae5227SSargun Dhillon 35796ae5227SSargun Dhillon static const struct bpf_func_proto *bpf_get_probe_write_proto(void) 35896ae5227SSargun Dhillon { 3592c78ee89SAlexei Starovoitov if (!capable(CAP_SYS_ADMIN)) 3602c78ee89SAlexei Starovoitov return NULL; 3612c78ee89SAlexei Starovoitov 36296ae5227SSargun Dhillon pr_warn_ratelimited("%s[%d] is installing a program with bpf_probe_write_user helper that may corrupt user memory!", 36396ae5227SSargun Dhillon current->comm, task_pid_nr(current)); 36496ae5227SSargun Dhillon 36596ae5227SSargun Dhillon return &bpf_probe_write_user_proto; 36696ae5227SSargun Dhillon } 36796ae5227SSargun Dhillon 368d7b2977bSChristoph Hellwig static void bpf_trace_copy_string(char *buf, void *unsafe_ptr, char fmt_ptype, 369d7b2977bSChristoph Hellwig size_t bufsz) 370d7b2977bSChristoph Hellwig { 371d7b2977bSChristoph Hellwig void __user *user_ptr = (__force void __user *)unsafe_ptr; 372d7b2977bSChristoph Hellwig 373d7b2977bSChristoph Hellwig buf[0] = 0; 374d7b2977bSChristoph Hellwig 375d7b2977bSChristoph Hellwig switch (fmt_ptype) { 376d7b2977bSChristoph Hellwig case 's': 377d7b2977bSChristoph Hellwig #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE 378aec6ce59SChristoph Hellwig if ((unsigned long)unsafe_ptr < TASK_SIZE) { 379aec6ce59SChristoph Hellwig strncpy_from_user_nofault(buf, user_ptr, bufsz); 380d7b2977bSChristoph Hellwig break; 381aec6ce59SChristoph Hellwig } 382aec6ce59SChristoph Hellwig fallthrough; 383d7b2977bSChristoph Hellwig #endif 384d7b2977bSChristoph Hellwig case 'k': 385d7b2977bSChristoph Hellwig strncpy_from_kernel_nofault(buf, unsafe_ptr, bufsz); 386d7b2977bSChristoph Hellwig break; 387d7b2977bSChristoph Hellwig case 'u': 388d7b2977bSChristoph Hellwig strncpy_from_user_nofault(buf, user_ptr, bufsz); 389d7b2977bSChristoph Hellwig break; 390d7b2977bSChristoph Hellwig } 391d7b2977bSChristoph Hellwig } 392d7b2977bSChristoph Hellwig 393ac5a72eaSAlan Maguire static DEFINE_RAW_SPINLOCK(trace_printk_lock); 394ac5a72eaSAlan Maguire 395ac5a72eaSAlan Maguire #define BPF_TRACE_PRINTK_SIZE 1024 396ac5a72eaSAlan Maguire 3970d360d64SStanislav Fomichev static __printf(1, 0) int bpf_do_trace_printk(const char *fmt, ...) 398ac5a72eaSAlan Maguire { 399ac5a72eaSAlan Maguire static char buf[BPF_TRACE_PRINTK_SIZE]; 400ac5a72eaSAlan Maguire unsigned long flags; 401ac5a72eaSAlan Maguire va_list ap; 402ac5a72eaSAlan Maguire int ret; 403ac5a72eaSAlan Maguire 404ac5a72eaSAlan Maguire raw_spin_lock_irqsave(&trace_printk_lock, flags); 405ac5a72eaSAlan Maguire va_start(ap, fmt); 406ac5a72eaSAlan Maguire ret = vsnprintf(buf, sizeof(buf), fmt, ap); 407ac5a72eaSAlan Maguire va_end(ap); 408ac5a72eaSAlan Maguire /* vsnprintf() will not append null for zero-length strings */ 409ac5a72eaSAlan Maguire if (ret == 0) 410ac5a72eaSAlan Maguire buf[0] = '\0'; 411ac5a72eaSAlan Maguire trace_bpf_trace_printk(buf); 412ac5a72eaSAlan Maguire raw_spin_unlock_irqrestore(&trace_printk_lock, flags); 413ac5a72eaSAlan Maguire 414ac5a72eaSAlan Maguire return ret; 415ac5a72eaSAlan Maguire } 416ac5a72eaSAlan Maguire 4179c959c86SAlexei Starovoitov /* 4187bda4b40SJohn Fastabend * Only limited trace_printk() conversion specifiers allowed: 4192df6bb54SSong Liu * %d %i %u %x %ld %li %lu %lx %lld %lli %llu %llx %p %pB %pks %pus %s 4209c959c86SAlexei Starovoitov */ 421f3694e00SDaniel Borkmann BPF_CALL_5(bpf_trace_printk, char *, fmt, u32, fmt_size, u64, arg1, 422f3694e00SDaniel Borkmann u64, arg2, u64, arg3) 4239c959c86SAlexei Starovoitov { 424b2a5212fSDaniel Borkmann int i, mod[3] = {}, fmt_cnt = 0; 425b2a5212fSDaniel Borkmann char buf[64], fmt_ptype; 426b2a5212fSDaniel Borkmann void *unsafe_ptr = NULL; 4278d3b7dceSAlexei Starovoitov bool str_seen = false; 4289c959c86SAlexei Starovoitov 4299c959c86SAlexei Starovoitov /* 4309c959c86SAlexei Starovoitov * bpf_check()->check_func_arg()->check_stack_boundary() 4319c959c86SAlexei Starovoitov * guarantees that fmt points to bpf program stack, 4329c959c86SAlexei Starovoitov * fmt_size bytes of it were initialized and fmt_size > 0 4339c959c86SAlexei Starovoitov */ 4349c959c86SAlexei Starovoitov if (fmt[--fmt_size] != 0) 4359c959c86SAlexei Starovoitov return -EINVAL; 4369c959c86SAlexei Starovoitov 4379c959c86SAlexei Starovoitov /* check format string for allowed specifiers */ 4389c959c86SAlexei Starovoitov for (i = 0; i < fmt_size; i++) { 4399c959c86SAlexei Starovoitov if ((!isprint(fmt[i]) && !isspace(fmt[i])) || !isascii(fmt[i])) 4409c959c86SAlexei Starovoitov return -EINVAL; 4419c959c86SAlexei Starovoitov 4429c959c86SAlexei Starovoitov if (fmt[i] != '%') 4439c959c86SAlexei Starovoitov continue; 4449c959c86SAlexei Starovoitov 4459c959c86SAlexei Starovoitov if (fmt_cnt >= 3) 4469c959c86SAlexei Starovoitov return -EINVAL; 4479c959c86SAlexei Starovoitov 4489c959c86SAlexei Starovoitov /* fmt[i] != 0 && fmt[last] == 0, so we can access fmt[i + 1] */ 4499c959c86SAlexei Starovoitov i++; 4509c959c86SAlexei Starovoitov if (fmt[i] == 'l') { 4519c959c86SAlexei Starovoitov mod[fmt_cnt]++; 4529c959c86SAlexei Starovoitov i++; 453b2a5212fSDaniel Borkmann } else if (fmt[i] == 'p') { 4549c959c86SAlexei Starovoitov mod[fmt_cnt]++; 455b2a5212fSDaniel Borkmann if ((fmt[i + 1] == 'k' || 456b2a5212fSDaniel Borkmann fmt[i + 1] == 'u') && 457b2a5212fSDaniel Borkmann fmt[i + 2] == 's') { 458b2a5212fSDaniel Borkmann fmt_ptype = fmt[i + 1]; 459b2a5212fSDaniel Borkmann i += 2; 460b2a5212fSDaniel Borkmann goto fmt_str; 461b2a5212fSDaniel Borkmann } 462b2a5212fSDaniel Borkmann 4632df6bb54SSong Liu if (fmt[i + 1] == 'B') { 4642df6bb54SSong Liu i++; 4652df6bb54SSong Liu goto fmt_next; 4662df6bb54SSong Liu } 4672df6bb54SSong Liu 4681efb6ee3SMartynas Pumputis /* disallow any further format extensions */ 4691efb6ee3SMartynas Pumputis if (fmt[i + 1] != 0 && 4701efb6ee3SMartynas Pumputis !isspace(fmt[i + 1]) && 4711efb6ee3SMartynas Pumputis !ispunct(fmt[i + 1])) 4729c959c86SAlexei Starovoitov return -EINVAL; 473b2a5212fSDaniel Borkmann 474b2a5212fSDaniel Borkmann goto fmt_next; 475b2a5212fSDaniel Borkmann } else if (fmt[i] == 's') { 476b2a5212fSDaniel Borkmann mod[fmt_cnt]++; 477b2a5212fSDaniel Borkmann fmt_ptype = fmt[i]; 478b2a5212fSDaniel Borkmann fmt_str: 4798d3b7dceSAlexei Starovoitov if (str_seen) 4808d3b7dceSAlexei Starovoitov /* allow only one '%s' per fmt string */ 4818d3b7dceSAlexei Starovoitov return -EINVAL; 4828d3b7dceSAlexei Starovoitov str_seen = true; 4838d3b7dceSAlexei Starovoitov 484b2a5212fSDaniel Borkmann if (fmt[i + 1] != 0 && 485b2a5212fSDaniel Borkmann !isspace(fmt[i + 1]) && 486b2a5212fSDaniel Borkmann !ispunct(fmt[i + 1])) 487b2a5212fSDaniel Borkmann return -EINVAL; 488b2a5212fSDaniel Borkmann 4898d3b7dceSAlexei Starovoitov switch (fmt_cnt) { 490b2a5212fSDaniel Borkmann case 0: 491b2a5212fSDaniel Borkmann unsafe_ptr = (void *)(long)arg1; 492f3694e00SDaniel Borkmann arg1 = (long)buf; 4938d3b7dceSAlexei Starovoitov break; 494b2a5212fSDaniel Borkmann case 1: 495b2a5212fSDaniel Borkmann unsafe_ptr = (void *)(long)arg2; 496f3694e00SDaniel Borkmann arg2 = (long)buf; 4978d3b7dceSAlexei Starovoitov break; 498b2a5212fSDaniel Borkmann case 2: 499b2a5212fSDaniel Borkmann unsafe_ptr = (void *)(long)arg3; 500f3694e00SDaniel Borkmann arg3 = (long)buf; 5018d3b7dceSAlexei Starovoitov break; 5028d3b7dceSAlexei Starovoitov } 503b2a5212fSDaniel Borkmann 504d7b2977bSChristoph Hellwig bpf_trace_copy_string(buf, unsafe_ptr, fmt_ptype, 5058d3b7dceSAlexei Starovoitov sizeof(buf)); 506b2a5212fSDaniel Borkmann goto fmt_next; 5079c959c86SAlexei Starovoitov } 5089c959c86SAlexei Starovoitov 5099c959c86SAlexei Starovoitov if (fmt[i] == 'l') { 5109c959c86SAlexei Starovoitov mod[fmt_cnt]++; 5119c959c86SAlexei Starovoitov i++; 5129c959c86SAlexei Starovoitov } 5139c959c86SAlexei Starovoitov 5147bda4b40SJohn Fastabend if (fmt[i] != 'i' && fmt[i] != 'd' && 5157bda4b40SJohn Fastabend fmt[i] != 'u' && fmt[i] != 'x') 5169c959c86SAlexei Starovoitov return -EINVAL; 517b2a5212fSDaniel Borkmann fmt_next: 5189c959c86SAlexei Starovoitov fmt_cnt++; 5199c959c86SAlexei Starovoitov } 5209c959c86SAlexei Starovoitov 52188a5c690SDaniel Borkmann /* Horrid workaround for getting va_list handling working with different 52288a5c690SDaniel Borkmann * argument type combinations generically for 32 and 64 bit archs. 52388a5c690SDaniel Borkmann */ 52488a5c690SDaniel Borkmann #define __BPF_TP_EMIT() __BPF_ARG3_TP() 52588a5c690SDaniel Borkmann #define __BPF_TP(...) \ 526ac5a72eaSAlan Maguire bpf_do_trace_printk(fmt, ##__VA_ARGS__) 52788a5c690SDaniel Borkmann 52888a5c690SDaniel Borkmann #define __BPF_ARG1_TP(...) \ 52988a5c690SDaniel Borkmann ((mod[0] == 2 || (mod[0] == 1 && __BITS_PER_LONG == 64)) \ 53088a5c690SDaniel Borkmann ? __BPF_TP(arg1, ##__VA_ARGS__) \ 53188a5c690SDaniel Borkmann : ((mod[0] == 1 || (mod[0] == 0 && __BITS_PER_LONG == 32)) \ 53288a5c690SDaniel Borkmann ? __BPF_TP((long)arg1, ##__VA_ARGS__) \ 53388a5c690SDaniel Borkmann : __BPF_TP((u32)arg1, ##__VA_ARGS__))) 53488a5c690SDaniel Borkmann 53588a5c690SDaniel Borkmann #define __BPF_ARG2_TP(...) \ 53688a5c690SDaniel Borkmann ((mod[1] == 2 || (mod[1] == 1 && __BITS_PER_LONG == 64)) \ 53788a5c690SDaniel Borkmann ? __BPF_ARG1_TP(arg2, ##__VA_ARGS__) \ 53888a5c690SDaniel Borkmann : ((mod[1] == 1 || (mod[1] == 0 && __BITS_PER_LONG == 32)) \ 53988a5c690SDaniel Borkmann ? __BPF_ARG1_TP((long)arg2, ##__VA_ARGS__) \ 54088a5c690SDaniel Borkmann : __BPF_ARG1_TP((u32)arg2, ##__VA_ARGS__))) 54188a5c690SDaniel Borkmann 54288a5c690SDaniel Borkmann #define __BPF_ARG3_TP(...) \ 54388a5c690SDaniel Borkmann ((mod[2] == 2 || (mod[2] == 1 && __BITS_PER_LONG == 64)) \ 54488a5c690SDaniel Borkmann ? __BPF_ARG2_TP(arg3, ##__VA_ARGS__) \ 54588a5c690SDaniel Borkmann : ((mod[2] == 1 || (mod[2] == 0 && __BITS_PER_LONG == 32)) \ 54688a5c690SDaniel Borkmann ? __BPF_ARG2_TP((long)arg3, ##__VA_ARGS__) \ 54788a5c690SDaniel Borkmann : __BPF_ARG2_TP((u32)arg3, ##__VA_ARGS__))) 54888a5c690SDaniel Borkmann 54988a5c690SDaniel Borkmann return __BPF_TP_EMIT(); 5509c959c86SAlexei Starovoitov } 5519c959c86SAlexei Starovoitov 5529c959c86SAlexei Starovoitov static const struct bpf_func_proto bpf_trace_printk_proto = { 5539c959c86SAlexei Starovoitov .func = bpf_trace_printk, 5549c959c86SAlexei Starovoitov .gpl_only = true, 5559c959c86SAlexei Starovoitov .ret_type = RET_INTEGER, 55639f19ebbSAlexei Starovoitov .arg1_type = ARG_PTR_TO_MEM, 55739f19ebbSAlexei Starovoitov .arg2_type = ARG_CONST_SIZE, 5589c959c86SAlexei Starovoitov }; 5599c959c86SAlexei Starovoitov 5600756ea3eSAlexei Starovoitov const struct bpf_func_proto *bpf_get_trace_printk_proto(void) 5610756ea3eSAlexei Starovoitov { 5620756ea3eSAlexei Starovoitov /* 563ac5a72eaSAlan Maguire * This program might be calling bpf_trace_printk, 564ac5a72eaSAlan Maguire * so enable the associated bpf_trace/bpf_trace_printk event. 565ac5a72eaSAlan Maguire * Repeat this each time as it is possible a user has 566ac5a72eaSAlan Maguire * disabled bpf_trace_printk events. By loading a program 567ac5a72eaSAlan Maguire * calling bpf_trace_printk() however the user has expressed 568ac5a72eaSAlan Maguire * the intent to see such events. 5690756ea3eSAlexei Starovoitov */ 570ac5a72eaSAlan Maguire if (trace_set_clr_event("bpf_trace", "bpf_trace_printk", 1)) 571ac5a72eaSAlan Maguire pr_warn_ratelimited("could not enable bpf_trace_printk events"); 5720756ea3eSAlexei Starovoitov 5730756ea3eSAlexei Starovoitov return &bpf_trace_printk_proto; 5740756ea3eSAlexei Starovoitov } 5750756ea3eSAlexei Starovoitov 576492e639fSYonghong Song #define MAX_SEQ_PRINTF_VARARGS 12 577492e639fSYonghong Song #define MAX_SEQ_PRINTF_MAX_MEMCPY 6 578492e639fSYonghong Song #define MAX_SEQ_PRINTF_STR_LEN 128 579492e639fSYonghong Song 580492e639fSYonghong Song struct bpf_seq_printf_buf { 581492e639fSYonghong Song char buf[MAX_SEQ_PRINTF_MAX_MEMCPY][MAX_SEQ_PRINTF_STR_LEN]; 582492e639fSYonghong Song }; 583492e639fSYonghong Song static DEFINE_PER_CPU(struct bpf_seq_printf_buf, bpf_seq_printf_buf); 584492e639fSYonghong Song static DEFINE_PER_CPU(int, bpf_seq_printf_buf_used); 585492e639fSYonghong Song 586492e639fSYonghong Song BPF_CALL_5(bpf_seq_printf, struct seq_file *, m, char *, fmt, u32, fmt_size, 587492e639fSYonghong Song const void *, data, u32, data_len) 588492e639fSYonghong Song { 589492e639fSYonghong Song int err = -EINVAL, fmt_cnt = 0, memcpy_cnt = 0; 590492e639fSYonghong Song int i, buf_used, copy_size, num_args; 591492e639fSYonghong Song u64 params[MAX_SEQ_PRINTF_VARARGS]; 592492e639fSYonghong Song struct bpf_seq_printf_buf *bufs; 593492e639fSYonghong Song const u64 *args = data; 594492e639fSYonghong Song 595492e639fSYonghong Song buf_used = this_cpu_inc_return(bpf_seq_printf_buf_used); 596492e639fSYonghong Song if (WARN_ON_ONCE(buf_used > 1)) { 597492e639fSYonghong Song err = -EBUSY; 598492e639fSYonghong Song goto out; 599492e639fSYonghong Song } 600492e639fSYonghong Song 601492e639fSYonghong Song bufs = this_cpu_ptr(&bpf_seq_printf_buf); 602492e639fSYonghong Song 603492e639fSYonghong Song /* 604492e639fSYonghong Song * bpf_check()->check_func_arg()->check_stack_boundary() 605492e639fSYonghong Song * guarantees that fmt points to bpf program stack, 606492e639fSYonghong Song * fmt_size bytes of it were initialized and fmt_size > 0 607492e639fSYonghong Song */ 608492e639fSYonghong Song if (fmt[--fmt_size] != 0) 609492e639fSYonghong Song goto out; 610492e639fSYonghong Song 611492e639fSYonghong Song if (data_len & 7) 612492e639fSYonghong Song goto out; 613492e639fSYonghong Song 614492e639fSYonghong Song for (i = 0; i < fmt_size; i++) { 615492e639fSYonghong Song if (fmt[i] == '%') { 616492e639fSYonghong Song if (fmt[i + 1] == '%') 617492e639fSYonghong Song i++; 618492e639fSYonghong Song else if (!data || !data_len) 619492e639fSYonghong Song goto out; 620492e639fSYonghong Song } 621492e639fSYonghong Song } 622492e639fSYonghong Song 623492e639fSYonghong Song num_args = data_len / 8; 624492e639fSYonghong Song 625492e639fSYonghong Song /* check format string for allowed specifiers */ 626492e639fSYonghong Song for (i = 0; i < fmt_size; i++) { 627492e639fSYonghong Song /* only printable ascii for now. */ 628492e639fSYonghong Song if ((!isprint(fmt[i]) && !isspace(fmt[i])) || !isascii(fmt[i])) { 629492e639fSYonghong Song err = -EINVAL; 630492e639fSYonghong Song goto out; 631492e639fSYonghong Song } 632492e639fSYonghong Song 633492e639fSYonghong Song if (fmt[i] != '%') 634492e639fSYonghong Song continue; 635492e639fSYonghong Song 636492e639fSYonghong Song if (fmt[i + 1] == '%') { 637492e639fSYonghong Song i++; 638492e639fSYonghong Song continue; 639492e639fSYonghong Song } 640492e639fSYonghong Song 641492e639fSYonghong Song if (fmt_cnt >= MAX_SEQ_PRINTF_VARARGS) { 642492e639fSYonghong Song err = -E2BIG; 643492e639fSYonghong Song goto out; 644492e639fSYonghong Song } 645492e639fSYonghong Song 646492e639fSYonghong Song if (fmt_cnt >= num_args) { 647492e639fSYonghong Song err = -EINVAL; 648492e639fSYonghong Song goto out; 649492e639fSYonghong Song } 650492e639fSYonghong Song 651492e639fSYonghong Song /* fmt[i] != 0 && fmt[last] == 0, so we can access fmt[i + 1] */ 652492e639fSYonghong Song i++; 653492e639fSYonghong Song 654492e639fSYonghong Song /* skip optional "[0 +-][num]" width formating field */ 655492e639fSYonghong Song while (fmt[i] == '0' || fmt[i] == '+' || fmt[i] == '-' || 656492e639fSYonghong Song fmt[i] == ' ') 657492e639fSYonghong Song i++; 658492e639fSYonghong Song if (fmt[i] >= '1' && fmt[i] <= '9') { 659492e639fSYonghong Song i++; 660492e639fSYonghong Song while (fmt[i] >= '0' && fmt[i] <= '9') 661492e639fSYonghong Song i++; 662492e639fSYonghong Song } 663492e639fSYonghong Song 664492e639fSYonghong Song if (fmt[i] == 's') { 66519c8d8acSAndrew Morton void *unsafe_ptr; 66619c8d8acSAndrew Morton 667492e639fSYonghong Song /* try our best to copy */ 668492e639fSYonghong Song if (memcpy_cnt >= MAX_SEQ_PRINTF_MAX_MEMCPY) { 669492e639fSYonghong Song err = -E2BIG; 670492e639fSYonghong Song goto out; 671492e639fSYonghong Song } 672492e639fSYonghong Song 67319c8d8acSAndrew Morton unsafe_ptr = (void *)(long)args[fmt_cnt]; 67419c8d8acSAndrew Morton err = strncpy_from_kernel_nofault(bufs->buf[memcpy_cnt], 67519c8d8acSAndrew Morton unsafe_ptr, MAX_SEQ_PRINTF_STR_LEN); 676492e639fSYonghong Song if (err < 0) 677492e639fSYonghong Song bufs->buf[memcpy_cnt][0] = '\0'; 678492e639fSYonghong Song params[fmt_cnt] = (u64)(long)bufs->buf[memcpy_cnt]; 679492e639fSYonghong Song 680492e639fSYonghong Song fmt_cnt++; 681492e639fSYonghong Song memcpy_cnt++; 682492e639fSYonghong Song continue; 683492e639fSYonghong Song } 684492e639fSYonghong Song 685492e639fSYonghong Song if (fmt[i] == 'p') { 686492e639fSYonghong Song if (fmt[i + 1] == 0 || 687492e639fSYonghong Song fmt[i + 1] == 'K' || 6882df6bb54SSong Liu fmt[i + 1] == 'x' || 6892df6bb54SSong Liu fmt[i + 1] == 'B') { 690492e639fSYonghong Song /* just kernel pointers */ 691492e639fSYonghong Song params[fmt_cnt] = args[fmt_cnt]; 692492e639fSYonghong Song fmt_cnt++; 693492e639fSYonghong Song continue; 694492e639fSYonghong Song } 695492e639fSYonghong Song 696492e639fSYonghong Song /* only support "%pI4", "%pi4", "%pI6" and "%pi6". */ 697492e639fSYonghong Song if (fmt[i + 1] != 'i' && fmt[i + 1] != 'I') { 698492e639fSYonghong Song err = -EINVAL; 699492e639fSYonghong Song goto out; 700492e639fSYonghong Song } 701492e639fSYonghong Song if (fmt[i + 2] != '4' && fmt[i + 2] != '6') { 702492e639fSYonghong Song err = -EINVAL; 703492e639fSYonghong Song goto out; 704492e639fSYonghong Song } 705492e639fSYonghong Song 706492e639fSYonghong Song if (memcpy_cnt >= MAX_SEQ_PRINTF_MAX_MEMCPY) { 707492e639fSYonghong Song err = -E2BIG; 708492e639fSYonghong Song goto out; 709492e639fSYonghong Song } 710492e639fSYonghong Song 711492e639fSYonghong Song 712492e639fSYonghong Song copy_size = (fmt[i + 2] == '4') ? 4 : 16; 713492e639fSYonghong Song 714fe557319SChristoph Hellwig err = copy_from_kernel_nofault(bufs->buf[memcpy_cnt], 715492e639fSYonghong Song (void *) (long) args[fmt_cnt], 716492e639fSYonghong Song copy_size); 717492e639fSYonghong Song if (err < 0) 718492e639fSYonghong Song memset(bufs->buf[memcpy_cnt], 0, copy_size); 719492e639fSYonghong Song params[fmt_cnt] = (u64)(long)bufs->buf[memcpy_cnt]; 720492e639fSYonghong Song 721492e639fSYonghong Song i += 2; 722492e639fSYonghong Song fmt_cnt++; 723492e639fSYonghong Song memcpy_cnt++; 724492e639fSYonghong Song continue; 725492e639fSYonghong Song } 726492e639fSYonghong Song 727492e639fSYonghong Song if (fmt[i] == 'l') { 728492e639fSYonghong Song i++; 729492e639fSYonghong Song if (fmt[i] == 'l') 730492e639fSYonghong Song i++; 731492e639fSYonghong Song } 732492e639fSYonghong Song 733492e639fSYonghong Song if (fmt[i] != 'i' && fmt[i] != 'd' && 734c06b0229SYonghong Song fmt[i] != 'u' && fmt[i] != 'x' && 735c06b0229SYonghong Song fmt[i] != 'X') { 736492e639fSYonghong Song err = -EINVAL; 737492e639fSYonghong Song goto out; 738492e639fSYonghong Song } 739492e639fSYonghong Song 740492e639fSYonghong Song params[fmt_cnt] = args[fmt_cnt]; 741492e639fSYonghong Song fmt_cnt++; 742492e639fSYonghong Song } 743492e639fSYonghong Song 744492e639fSYonghong Song /* Maximumly we can have MAX_SEQ_PRINTF_VARARGS parameter, just give 745492e639fSYonghong Song * all of them to seq_printf(). 746492e639fSYonghong Song */ 747492e639fSYonghong Song seq_printf(m, fmt, params[0], params[1], params[2], params[3], 748492e639fSYonghong Song params[4], params[5], params[6], params[7], params[8], 749492e639fSYonghong Song params[9], params[10], params[11]); 750492e639fSYonghong Song 751492e639fSYonghong Song err = seq_has_overflowed(m) ? -EOVERFLOW : 0; 752492e639fSYonghong Song out: 753492e639fSYonghong Song this_cpu_dec(bpf_seq_printf_buf_used); 754492e639fSYonghong Song return err; 755492e639fSYonghong Song } 756492e639fSYonghong Song 7579436ef6eSLorenz Bauer BTF_ID_LIST_SINGLE(btf_seq_file_ids, struct, seq_file) 758c9a0f3b8SJiri Olsa 759492e639fSYonghong Song static const struct bpf_func_proto bpf_seq_printf_proto = { 760492e639fSYonghong Song .func = bpf_seq_printf, 761492e639fSYonghong Song .gpl_only = true, 762492e639fSYonghong Song .ret_type = RET_INTEGER, 763492e639fSYonghong Song .arg1_type = ARG_PTR_TO_BTF_ID, 7649436ef6eSLorenz Bauer .arg1_btf_id = &btf_seq_file_ids[0], 765492e639fSYonghong Song .arg2_type = ARG_PTR_TO_MEM, 766492e639fSYonghong Song .arg3_type = ARG_CONST_SIZE, 767492e639fSYonghong Song .arg4_type = ARG_PTR_TO_MEM_OR_NULL, 768492e639fSYonghong Song .arg5_type = ARG_CONST_SIZE_OR_ZERO, 769492e639fSYonghong Song }; 770492e639fSYonghong Song 771492e639fSYonghong Song BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len) 772492e639fSYonghong Song { 773492e639fSYonghong Song return seq_write(m, data, len) ? -EOVERFLOW : 0; 774492e639fSYonghong Song } 775492e639fSYonghong Song 776492e639fSYonghong Song static const struct bpf_func_proto bpf_seq_write_proto = { 777492e639fSYonghong Song .func = bpf_seq_write, 778492e639fSYonghong Song .gpl_only = true, 779492e639fSYonghong Song .ret_type = RET_INTEGER, 780492e639fSYonghong Song .arg1_type = ARG_PTR_TO_BTF_ID, 7819436ef6eSLorenz Bauer .arg1_btf_id = &btf_seq_file_ids[0], 782492e639fSYonghong Song .arg2_type = ARG_PTR_TO_MEM, 783492e639fSYonghong Song .arg3_type = ARG_CONST_SIZE_OR_ZERO, 784492e639fSYonghong Song }; 785492e639fSYonghong Song 786eb411377SAlan Maguire BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr, 787eb411377SAlan Maguire u32, btf_ptr_size, u64, flags) 788eb411377SAlan Maguire { 789eb411377SAlan Maguire const struct btf *btf; 790eb411377SAlan Maguire s32 btf_id; 791eb411377SAlan Maguire int ret; 792eb411377SAlan Maguire 793eb411377SAlan Maguire ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id); 794eb411377SAlan Maguire if (ret) 795eb411377SAlan Maguire return ret; 796eb411377SAlan Maguire 797eb411377SAlan Maguire return btf_type_seq_show_flags(btf, btf_id, ptr->ptr, m, flags); 798eb411377SAlan Maguire } 799eb411377SAlan Maguire 800eb411377SAlan Maguire static const struct bpf_func_proto bpf_seq_printf_btf_proto = { 801eb411377SAlan Maguire .func = bpf_seq_printf_btf, 802eb411377SAlan Maguire .gpl_only = true, 803eb411377SAlan Maguire .ret_type = RET_INTEGER, 804eb411377SAlan Maguire .arg1_type = ARG_PTR_TO_BTF_ID, 805eb411377SAlan Maguire .arg1_btf_id = &btf_seq_file_ids[0], 806eb411377SAlan Maguire .arg2_type = ARG_PTR_TO_MEM, 807eb411377SAlan Maguire .arg3_type = ARG_CONST_SIZE_OR_ZERO, 808eb411377SAlan Maguire .arg4_type = ARG_ANYTHING, 809d9847d31SAlexei Starovoitov }; 810d9847d31SAlexei Starovoitov 811908432caSYonghong Song static __always_inline int 812908432caSYonghong Song get_map_perf_counter(struct bpf_map *map, u64 flags, 813908432caSYonghong Song u64 *value, u64 *enabled, u64 *running) 81435578d79SKaixu Xia { 81535578d79SKaixu Xia struct bpf_array *array = container_of(map, struct bpf_array, map); 8166816a7ffSDaniel Borkmann unsigned int cpu = smp_processor_id(); 8176816a7ffSDaniel Borkmann u64 index = flags & BPF_F_INDEX_MASK; 8183b1efb19SDaniel Borkmann struct bpf_event_entry *ee; 81935578d79SKaixu Xia 8206816a7ffSDaniel Borkmann if (unlikely(flags & ~(BPF_F_INDEX_MASK))) 8216816a7ffSDaniel Borkmann return -EINVAL; 8226816a7ffSDaniel Borkmann if (index == BPF_F_CURRENT_CPU) 8236816a7ffSDaniel Borkmann index = cpu; 82435578d79SKaixu Xia if (unlikely(index >= array->map.max_entries)) 82535578d79SKaixu Xia return -E2BIG; 82635578d79SKaixu Xia 8273b1efb19SDaniel Borkmann ee = READ_ONCE(array->ptrs[index]); 8281ca1cc98SDaniel Borkmann if (!ee) 82935578d79SKaixu Xia return -ENOENT; 83035578d79SKaixu Xia 831908432caSYonghong Song return perf_event_read_local(ee->event, value, enabled, running); 832908432caSYonghong Song } 833908432caSYonghong Song 834908432caSYonghong Song BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags) 835908432caSYonghong Song { 836908432caSYonghong Song u64 value = 0; 837908432caSYonghong Song int err; 838908432caSYonghong Song 839908432caSYonghong Song err = get_map_perf_counter(map, flags, &value, NULL, NULL); 84035578d79SKaixu Xia /* 841f91840a3SAlexei Starovoitov * this api is ugly since we miss [-22..-2] range of valid 842f91840a3SAlexei Starovoitov * counter values, but that's uapi 84335578d79SKaixu Xia */ 844f91840a3SAlexei Starovoitov if (err) 845f91840a3SAlexei Starovoitov return err; 846f91840a3SAlexei Starovoitov return value; 84735578d79SKaixu Xia } 84835578d79SKaixu Xia 84962544ce8SAlexei Starovoitov static const struct bpf_func_proto bpf_perf_event_read_proto = { 85035578d79SKaixu Xia .func = bpf_perf_event_read, 8511075ef59SAlexei Starovoitov .gpl_only = true, 85235578d79SKaixu Xia .ret_type = RET_INTEGER, 85335578d79SKaixu Xia .arg1_type = ARG_CONST_MAP_PTR, 85435578d79SKaixu Xia .arg2_type = ARG_ANYTHING, 85535578d79SKaixu Xia }; 85635578d79SKaixu Xia 857908432caSYonghong Song BPF_CALL_4(bpf_perf_event_read_value, struct bpf_map *, map, u64, flags, 858908432caSYonghong Song struct bpf_perf_event_value *, buf, u32, size) 859908432caSYonghong Song { 860908432caSYonghong Song int err = -EINVAL; 861908432caSYonghong Song 862908432caSYonghong Song if (unlikely(size != sizeof(struct bpf_perf_event_value))) 863908432caSYonghong Song goto clear; 864908432caSYonghong Song err = get_map_perf_counter(map, flags, &buf->counter, &buf->enabled, 865908432caSYonghong Song &buf->running); 866908432caSYonghong Song if (unlikely(err)) 867908432caSYonghong Song goto clear; 868908432caSYonghong Song return 0; 869908432caSYonghong Song clear: 870908432caSYonghong Song memset(buf, 0, size); 871908432caSYonghong Song return err; 872908432caSYonghong Song } 873908432caSYonghong Song 874908432caSYonghong Song static const struct bpf_func_proto bpf_perf_event_read_value_proto = { 875908432caSYonghong Song .func = bpf_perf_event_read_value, 876908432caSYonghong Song .gpl_only = true, 877908432caSYonghong Song .ret_type = RET_INTEGER, 878908432caSYonghong Song .arg1_type = ARG_CONST_MAP_PTR, 879908432caSYonghong Song .arg2_type = ARG_ANYTHING, 880908432caSYonghong Song .arg3_type = ARG_PTR_TO_UNINIT_MEM, 881908432caSYonghong Song .arg4_type = ARG_CONST_SIZE, 882908432caSYonghong Song }; 883908432caSYonghong Song 8848e7a3920SDaniel Borkmann static __always_inline u64 8858e7a3920SDaniel Borkmann __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, 886283ca526SDaniel Borkmann u64 flags, struct perf_sample_data *sd) 887a43eec30SAlexei Starovoitov { 888a43eec30SAlexei Starovoitov struct bpf_array *array = container_of(map, struct bpf_array, map); 889d7931330SDaniel Borkmann unsigned int cpu = smp_processor_id(); 8901e33759cSDaniel Borkmann u64 index = flags & BPF_F_INDEX_MASK; 8913b1efb19SDaniel Borkmann struct bpf_event_entry *ee; 892a43eec30SAlexei Starovoitov struct perf_event *event; 893a43eec30SAlexei Starovoitov 8941e33759cSDaniel Borkmann if (index == BPF_F_CURRENT_CPU) 895d7931330SDaniel Borkmann index = cpu; 896a43eec30SAlexei Starovoitov if (unlikely(index >= array->map.max_entries)) 897a43eec30SAlexei Starovoitov return -E2BIG; 898a43eec30SAlexei Starovoitov 8993b1efb19SDaniel Borkmann ee = READ_ONCE(array->ptrs[index]); 9001ca1cc98SDaniel Borkmann if (!ee) 901a43eec30SAlexei Starovoitov return -ENOENT; 902a43eec30SAlexei Starovoitov 9033b1efb19SDaniel Borkmann event = ee->event; 904a43eec30SAlexei Starovoitov if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE || 905a43eec30SAlexei Starovoitov event->attr.config != PERF_COUNT_SW_BPF_OUTPUT)) 906a43eec30SAlexei Starovoitov return -EINVAL; 907a43eec30SAlexei Starovoitov 908d7931330SDaniel Borkmann if (unlikely(event->oncpu != cpu)) 909a43eec30SAlexei Starovoitov return -EOPNOTSUPP; 910a43eec30SAlexei Starovoitov 91156201969SArnaldo Carvalho de Melo return perf_event_output(event, sd, regs); 912a43eec30SAlexei Starovoitov } 913a43eec30SAlexei Starovoitov 9149594dc3cSMatt Mullins /* 9159594dc3cSMatt Mullins * Support executing tracepoints in normal, irq, and nmi context that each call 9169594dc3cSMatt Mullins * bpf_perf_event_output 9179594dc3cSMatt Mullins */ 9189594dc3cSMatt Mullins struct bpf_trace_sample_data { 9199594dc3cSMatt Mullins struct perf_sample_data sds[3]; 9209594dc3cSMatt Mullins }; 9219594dc3cSMatt Mullins 9229594dc3cSMatt Mullins static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds); 9239594dc3cSMatt Mullins static DEFINE_PER_CPU(int, bpf_trace_nest_level); 924f3694e00SDaniel Borkmann BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map, 925f3694e00SDaniel Borkmann u64, flags, void *, data, u64, size) 9268e7a3920SDaniel Borkmann { 9279594dc3cSMatt Mullins struct bpf_trace_sample_data *sds = this_cpu_ptr(&bpf_trace_sds); 9289594dc3cSMatt Mullins int nest_level = this_cpu_inc_return(bpf_trace_nest_level); 9298e7a3920SDaniel Borkmann struct perf_raw_record raw = { 9308e7a3920SDaniel Borkmann .frag = { 9318e7a3920SDaniel Borkmann .size = size, 9328e7a3920SDaniel Borkmann .data = data, 9338e7a3920SDaniel Borkmann }, 9348e7a3920SDaniel Borkmann }; 9359594dc3cSMatt Mullins struct perf_sample_data *sd; 9369594dc3cSMatt Mullins int err; 9378e7a3920SDaniel Borkmann 9389594dc3cSMatt Mullins if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds))) { 9399594dc3cSMatt Mullins err = -EBUSY; 9409594dc3cSMatt Mullins goto out; 9419594dc3cSMatt Mullins } 9429594dc3cSMatt Mullins 9439594dc3cSMatt Mullins sd = &sds->sds[nest_level - 1]; 9449594dc3cSMatt Mullins 9459594dc3cSMatt Mullins if (unlikely(flags & ~(BPF_F_INDEX_MASK))) { 9469594dc3cSMatt Mullins err = -EINVAL; 9479594dc3cSMatt Mullins goto out; 9489594dc3cSMatt Mullins } 9498e7a3920SDaniel Borkmann 950283ca526SDaniel Borkmann perf_sample_data_init(sd, 0, 0); 951283ca526SDaniel Borkmann sd->raw = &raw; 952283ca526SDaniel Borkmann 9539594dc3cSMatt Mullins err = __bpf_perf_event_output(regs, map, flags, sd); 9549594dc3cSMatt Mullins 9559594dc3cSMatt Mullins out: 9569594dc3cSMatt Mullins this_cpu_dec(bpf_trace_nest_level); 9579594dc3cSMatt Mullins return err; 9588e7a3920SDaniel Borkmann } 9598e7a3920SDaniel Borkmann 960a43eec30SAlexei Starovoitov static const struct bpf_func_proto bpf_perf_event_output_proto = { 961a43eec30SAlexei Starovoitov .func = bpf_perf_event_output, 9621075ef59SAlexei Starovoitov .gpl_only = true, 963a43eec30SAlexei Starovoitov .ret_type = RET_INTEGER, 964a43eec30SAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 965a43eec30SAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 966a43eec30SAlexei Starovoitov .arg3_type = ARG_ANYTHING, 96739f19ebbSAlexei Starovoitov .arg4_type = ARG_PTR_TO_MEM, 968a60dd35dSGianluca Borello .arg5_type = ARG_CONST_SIZE_OR_ZERO, 969a43eec30SAlexei Starovoitov }; 970a43eec30SAlexei Starovoitov 971768fb61fSAllan Zhang static DEFINE_PER_CPU(int, bpf_event_output_nest_level); 972768fb61fSAllan Zhang struct bpf_nested_pt_regs { 973768fb61fSAllan Zhang struct pt_regs regs[3]; 974768fb61fSAllan Zhang }; 975768fb61fSAllan Zhang static DEFINE_PER_CPU(struct bpf_nested_pt_regs, bpf_pt_regs); 976768fb61fSAllan Zhang static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_misc_sds); 977bd570ff9SDaniel Borkmann 978555c8a86SDaniel Borkmann u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 979555c8a86SDaniel Borkmann void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy) 980bd570ff9SDaniel Borkmann { 981768fb61fSAllan Zhang int nest_level = this_cpu_inc_return(bpf_event_output_nest_level); 982555c8a86SDaniel Borkmann struct perf_raw_frag frag = { 983555c8a86SDaniel Borkmann .copy = ctx_copy, 984555c8a86SDaniel Borkmann .size = ctx_size, 985555c8a86SDaniel Borkmann .data = ctx, 986555c8a86SDaniel Borkmann }; 987555c8a86SDaniel Borkmann struct perf_raw_record raw = { 988555c8a86SDaniel Borkmann .frag = { 989183fc153SAndrew Morton { 990555c8a86SDaniel Borkmann .next = ctx_size ? &frag : NULL, 991183fc153SAndrew Morton }, 992555c8a86SDaniel Borkmann .size = meta_size, 993555c8a86SDaniel Borkmann .data = meta, 994555c8a86SDaniel Borkmann }, 995555c8a86SDaniel Borkmann }; 996768fb61fSAllan Zhang struct perf_sample_data *sd; 997768fb61fSAllan Zhang struct pt_regs *regs; 998768fb61fSAllan Zhang u64 ret; 999768fb61fSAllan Zhang 1000768fb61fSAllan Zhang if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bpf_misc_sds.sds))) { 1001768fb61fSAllan Zhang ret = -EBUSY; 1002768fb61fSAllan Zhang goto out; 1003768fb61fSAllan Zhang } 1004768fb61fSAllan Zhang sd = this_cpu_ptr(&bpf_misc_sds.sds[nest_level - 1]); 1005768fb61fSAllan Zhang regs = this_cpu_ptr(&bpf_pt_regs.regs[nest_level - 1]); 1006bd570ff9SDaniel Borkmann 1007bd570ff9SDaniel Borkmann perf_fetch_caller_regs(regs); 1008283ca526SDaniel Borkmann perf_sample_data_init(sd, 0, 0); 1009283ca526SDaniel Borkmann sd->raw = &raw; 1010bd570ff9SDaniel Borkmann 1011768fb61fSAllan Zhang ret = __bpf_perf_event_output(regs, map, flags, sd); 1012768fb61fSAllan Zhang out: 1013768fb61fSAllan Zhang this_cpu_dec(bpf_event_output_nest_level); 1014768fb61fSAllan Zhang return ret; 1015bd570ff9SDaniel Borkmann } 1016bd570ff9SDaniel Borkmann 1017f3694e00SDaniel Borkmann BPF_CALL_0(bpf_get_current_task) 1018606274c5SAlexei Starovoitov { 1019606274c5SAlexei Starovoitov return (long) current; 1020606274c5SAlexei Starovoitov } 1021606274c5SAlexei Starovoitov 1022f470378cSJohn Fastabend const struct bpf_func_proto bpf_get_current_task_proto = { 1023606274c5SAlexei Starovoitov .func = bpf_get_current_task, 1024606274c5SAlexei Starovoitov .gpl_only = true, 1025606274c5SAlexei Starovoitov .ret_type = RET_INTEGER, 1026606274c5SAlexei Starovoitov }; 1027606274c5SAlexei Starovoitov 10283ca1032aSKP Singh BPF_CALL_0(bpf_get_current_task_btf) 10293ca1032aSKP Singh { 10303ca1032aSKP Singh return (unsigned long) current; 10313ca1032aSKP Singh } 10323ca1032aSKP Singh 10333ca1032aSKP Singh BTF_ID_LIST_SINGLE(bpf_get_current_btf_ids, struct, task_struct) 10343ca1032aSKP Singh 10353ca1032aSKP Singh static const struct bpf_func_proto bpf_get_current_task_btf_proto = { 10363ca1032aSKP Singh .func = bpf_get_current_task_btf, 10373ca1032aSKP Singh .gpl_only = true, 10383ca1032aSKP Singh .ret_type = RET_PTR_TO_BTF_ID, 10393ca1032aSKP Singh .ret_btf_id = &bpf_get_current_btf_ids[0], 10403ca1032aSKP Singh }; 10413ca1032aSKP Singh 1042f3694e00SDaniel Borkmann BPF_CALL_2(bpf_current_task_under_cgroup, struct bpf_map *, map, u32, idx) 104360d20f91SSargun Dhillon { 104460d20f91SSargun Dhillon struct bpf_array *array = container_of(map, struct bpf_array, map); 104560d20f91SSargun Dhillon struct cgroup *cgrp; 104660d20f91SSargun Dhillon 104760d20f91SSargun Dhillon if (unlikely(idx >= array->map.max_entries)) 104860d20f91SSargun Dhillon return -E2BIG; 104960d20f91SSargun Dhillon 105060d20f91SSargun Dhillon cgrp = READ_ONCE(array->ptrs[idx]); 105160d20f91SSargun Dhillon if (unlikely(!cgrp)) 105260d20f91SSargun Dhillon return -EAGAIN; 105360d20f91SSargun Dhillon 105460d20f91SSargun Dhillon return task_under_cgroup_hierarchy(current, cgrp); 105560d20f91SSargun Dhillon } 105660d20f91SSargun Dhillon 105760d20f91SSargun Dhillon static const struct bpf_func_proto bpf_current_task_under_cgroup_proto = { 105860d20f91SSargun Dhillon .func = bpf_current_task_under_cgroup, 105960d20f91SSargun Dhillon .gpl_only = false, 106060d20f91SSargun Dhillon .ret_type = RET_INTEGER, 106160d20f91SSargun Dhillon .arg1_type = ARG_CONST_MAP_PTR, 106260d20f91SSargun Dhillon .arg2_type = ARG_ANYTHING, 106360d20f91SSargun Dhillon }; 106460d20f91SSargun Dhillon 10658b401f9eSYonghong Song struct send_signal_irq_work { 10668b401f9eSYonghong Song struct irq_work irq_work; 10678b401f9eSYonghong Song struct task_struct *task; 10688b401f9eSYonghong Song u32 sig; 10698482941fSYonghong Song enum pid_type type; 10708b401f9eSYonghong Song }; 10718b401f9eSYonghong Song 10728b401f9eSYonghong Song static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work); 10738b401f9eSYonghong Song 10748b401f9eSYonghong Song static void do_bpf_send_signal(struct irq_work *entry) 10758b401f9eSYonghong Song { 10768b401f9eSYonghong Song struct send_signal_irq_work *work; 10778b401f9eSYonghong Song 10788b401f9eSYonghong Song work = container_of(entry, struct send_signal_irq_work, irq_work); 10798482941fSYonghong Song group_send_sig_info(work->sig, SEND_SIG_PRIV, work->task, work->type); 10808b401f9eSYonghong Song } 10818b401f9eSYonghong Song 10828482941fSYonghong Song static int bpf_send_signal_common(u32 sig, enum pid_type type) 10838b401f9eSYonghong Song { 10848b401f9eSYonghong Song struct send_signal_irq_work *work = NULL; 10858b401f9eSYonghong Song 10868b401f9eSYonghong Song /* Similar to bpf_probe_write_user, task needs to be 10878b401f9eSYonghong Song * in a sound condition and kernel memory access be 10888b401f9eSYonghong Song * permitted in order to send signal to the current 10898b401f9eSYonghong Song * task. 10908b401f9eSYonghong Song */ 10918b401f9eSYonghong Song if (unlikely(current->flags & (PF_KTHREAD | PF_EXITING))) 10928b401f9eSYonghong Song return -EPERM; 10938b401f9eSYonghong Song if (unlikely(uaccess_kernel())) 10948b401f9eSYonghong Song return -EPERM; 10958b401f9eSYonghong Song if (unlikely(!nmi_uaccess_okay())) 10968b401f9eSYonghong Song return -EPERM; 10978b401f9eSYonghong Song 10981bc7896eSYonghong Song if (irqs_disabled()) { 1099e1afb702SYonghong Song /* Do an early check on signal validity. Otherwise, 1100e1afb702SYonghong Song * the error is lost in deferred irq_work. 1101e1afb702SYonghong Song */ 1102e1afb702SYonghong Song if (unlikely(!valid_signal(sig))) 1103e1afb702SYonghong Song return -EINVAL; 1104e1afb702SYonghong Song 11058b401f9eSYonghong Song work = this_cpu_ptr(&send_signal_work); 1106153bedbaSFrederic Weisbecker if (atomic_read(&work->irq_work.flags) & IRQ_WORK_BUSY) 11078b401f9eSYonghong Song return -EBUSY; 11088b401f9eSYonghong Song 11098b401f9eSYonghong Song /* Add the current task, which is the target of sending signal, 11108b401f9eSYonghong Song * to the irq_work. The current task may change when queued 11118b401f9eSYonghong Song * irq works get executed. 11128b401f9eSYonghong Song */ 11138b401f9eSYonghong Song work->task = current; 11148b401f9eSYonghong Song work->sig = sig; 11158482941fSYonghong Song work->type = type; 11168b401f9eSYonghong Song irq_work_queue(&work->irq_work); 11178b401f9eSYonghong Song return 0; 11188b401f9eSYonghong Song } 11198b401f9eSYonghong Song 11208482941fSYonghong Song return group_send_sig_info(sig, SEND_SIG_PRIV, current, type); 11218482941fSYonghong Song } 11228482941fSYonghong Song 11238482941fSYonghong Song BPF_CALL_1(bpf_send_signal, u32, sig) 11248482941fSYonghong Song { 11258482941fSYonghong Song return bpf_send_signal_common(sig, PIDTYPE_TGID); 11268b401f9eSYonghong Song } 11278b401f9eSYonghong Song 11288b401f9eSYonghong Song static const struct bpf_func_proto bpf_send_signal_proto = { 11298b401f9eSYonghong Song .func = bpf_send_signal, 11308b401f9eSYonghong Song .gpl_only = false, 11318b401f9eSYonghong Song .ret_type = RET_INTEGER, 11328b401f9eSYonghong Song .arg1_type = ARG_ANYTHING, 11338b401f9eSYonghong Song }; 11348b401f9eSYonghong Song 11358482941fSYonghong Song BPF_CALL_1(bpf_send_signal_thread, u32, sig) 11368482941fSYonghong Song { 11378482941fSYonghong Song return bpf_send_signal_common(sig, PIDTYPE_PID); 11388482941fSYonghong Song } 11398482941fSYonghong Song 11408482941fSYonghong Song static const struct bpf_func_proto bpf_send_signal_thread_proto = { 11418482941fSYonghong Song .func = bpf_send_signal_thread, 11428482941fSYonghong Song .gpl_only = false, 11438482941fSYonghong Song .ret_type = RET_INTEGER, 11448482941fSYonghong Song .arg1_type = ARG_ANYTHING, 11458482941fSYonghong Song }; 11468482941fSYonghong Song 11476e22ab9dSJiri Olsa BPF_CALL_3(bpf_d_path, struct path *, path, char *, buf, u32, sz) 11486e22ab9dSJiri Olsa { 11496e22ab9dSJiri Olsa long len; 11506e22ab9dSJiri Olsa char *p; 11516e22ab9dSJiri Olsa 11526e22ab9dSJiri Olsa if (!sz) 11536e22ab9dSJiri Olsa return 0; 11546e22ab9dSJiri Olsa 11556e22ab9dSJiri Olsa p = d_path(path, buf, sz); 11566e22ab9dSJiri Olsa if (IS_ERR(p)) { 11576e22ab9dSJiri Olsa len = PTR_ERR(p); 11586e22ab9dSJiri Olsa } else { 11596e22ab9dSJiri Olsa len = buf + sz - p; 11606e22ab9dSJiri Olsa memmove(buf, p, len); 11616e22ab9dSJiri Olsa } 11626e22ab9dSJiri Olsa 11636e22ab9dSJiri Olsa return len; 11646e22ab9dSJiri Olsa } 11656e22ab9dSJiri Olsa 11666e22ab9dSJiri Olsa BTF_SET_START(btf_allowlist_d_path) 1167a8a71796SJiri Olsa #ifdef CONFIG_SECURITY 1168a8a71796SJiri Olsa BTF_ID(func, security_file_permission) 1169a8a71796SJiri Olsa BTF_ID(func, security_inode_getattr) 1170a8a71796SJiri Olsa BTF_ID(func, security_file_open) 1171a8a71796SJiri Olsa #endif 1172a8a71796SJiri Olsa #ifdef CONFIG_SECURITY_PATH 1173a8a71796SJiri Olsa BTF_ID(func, security_path_truncate) 1174a8a71796SJiri Olsa #endif 11756e22ab9dSJiri Olsa BTF_ID(func, vfs_truncate) 11766e22ab9dSJiri Olsa BTF_ID(func, vfs_fallocate) 11776e22ab9dSJiri Olsa BTF_ID(func, dentry_open) 11786e22ab9dSJiri Olsa BTF_ID(func, vfs_getattr) 11796e22ab9dSJiri Olsa BTF_ID(func, filp_close) 11806e22ab9dSJiri Olsa BTF_SET_END(btf_allowlist_d_path) 11816e22ab9dSJiri Olsa 11826e22ab9dSJiri Olsa static bool bpf_d_path_allowed(const struct bpf_prog *prog) 11836e22ab9dSJiri Olsa { 11846f100640SKP Singh if (prog->type == BPF_PROG_TYPE_LSM) 11856f100640SKP Singh return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id); 11866f100640SKP Singh 11876f100640SKP Singh return btf_id_set_contains(&btf_allowlist_d_path, 11886f100640SKP Singh prog->aux->attach_btf_id); 11896e22ab9dSJiri Olsa } 11906e22ab9dSJiri Olsa 11919436ef6eSLorenz Bauer BTF_ID_LIST_SINGLE(bpf_d_path_btf_ids, struct, path) 11926e22ab9dSJiri Olsa 11936e22ab9dSJiri Olsa static const struct bpf_func_proto bpf_d_path_proto = { 11946e22ab9dSJiri Olsa .func = bpf_d_path, 11956e22ab9dSJiri Olsa .gpl_only = false, 11966e22ab9dSJiri Olsa .ret_type = RET_INTEGER, 11976e22ab9dSJiri Olsa .arg1_type = ARG_PTR_TO_BTF_ID, 11989436ef6eSLorenz Bauer .arg1_btf_id = &bpf_d_path_btf_ids[0], 11996e22ab9dSJiri Olsa .arg2_type = ARG_PTR_TO_MEM, 12006e22ab9dSJiri Olsa .arg3_type = ARG_CONST_SIZE_OR_ZERO, 12016e22ab9dSJiri Olsa .allowed = bpf_d_path_allowed, 12026e22ab9dSJiri Olsa }; 12036e22ab9dSJiri Olsa 1204c4d0bfb4SAlan Maguire #define BTF_F_ALL (BTF_F_COMPACT | BTF_F_NONAME | \ 1205c4d0bfb4SAlan Maguire BTF_F_PTR_RAW | BTF_F_ZERO) 1206c4d0bfb4SAlan Maguire 1207c4d0bfb4SAlan Maguire static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size, 1208c4d0bfb4SAlan Maguire u64 flags, const struct btf **btf, 1209c4d0bfb4SAlan Maguire s32 *btf_id) 1210c4d0bfb4SAlan Maguire { 1211c4d0bfb4SAlan Maguire const struct btf_type *t; 1212c4d0bfb4SAlan Maguire 1213c4d0bfb4SAlan Maguire if (unlikely(flags & ~(BTF_F_ALL))) 1214c4d0bfb4SAlan Maguire return -EINVAL; 1215c4d0bfb4SAlan Maguire 1216c4d0bfb4SAlan Maguire if (btf_ptr_size != sizeof(struct btf_ptr)) 1217c4d0bfb4SAlan Maguire return -EINVAL; 1218c4d0bfb4SAlan Maguire 1219c4d0bfb4SAlan Maguire *btf = bpf_get_btf_vmlinux(); 1220c4d0bfb4SAlan Maguire 1221c4d0bfb4SAlan Maguire if (IS_ERR_OR_NULL(*btf)) 1222c4d0bfb4SAlan Maguire return PTR_ERR(*btf); 1223c4d0bfb4SAlan Maguire 1224c4d0bfb4SAlan Maguire if (ptr->type_id > 0) 1225c4d0bfb4SAlan Maguire *btf_id = ptr->type_id; 1226c4d0bfb4SAlan Maguire else 1227c4d0bfb4SAlan Maguire return -EINVAL; 1228c4d0bfb4SAlan Maguire 1229c4d0bfb4SAlan Maguire if (*btf_id > 0) 1230c4d0bfb4SAlan Maguire t = btf_type_by_id(*btf, *btf_id); 1231c4d0bfb4SAlan Maguire if (*btf_id <= 0 || !t) 1232c4d0bfb4SAlan Maguire return -ENOENT; 1233c4d0bfb4SAlan Maguire 1234c4d0bfb4SAlan Maguire return 0; 1235c4d0bfb4SAlan Maguire } 1236c4d0bfb4SAlan Maguire 1237c4d0bfb4SAlan Maguire BPF_CALL_5(bpf_snprintf_btf, char *, str, u32, str_size, struct btf_ptr *, ptr, 1238c4d0bfb4SAlan Maguire u32, btf_ptr_size, u64, flags) 1239c4d0bfb4SAlan Maguire { 1240c4d0bfb4SAlan Maguire const struct btf *btf; 1241c4d0bfb4SAlan Maguire s32 btf_id; 1242c4d0bfb4SAlan Maguire int ret; 1243c4d0bfb4SAlan Maguire 1244c4d0bfb4SAlan Maguire ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id); 1245c4d0bfb4SAlan Maguire if (ret) 1246c4d0bfb4SAlan Maguire return ret; 1247c4d0bfb4SAlan Maguire 1248c4d0bfb4SAlan Maguire return btf_type_snprintf_show(btf, btf_id, ptr->ptr, str, str_size, 1249c4d0bfb4SAlan Maguire flags); 1250c4d0bfb4SAlan Maguire } 1251c4d0bfb4SAlan Maguire 1252c4d0bfb4SAlan Maguire const struct bpf_func_proto bpf_snprintf_btf_proto = { 1253c4d0bfb4SAlan Maguire .func = bpf_snprintf_btf, 1254c4d0bfb4SAlan Maguire .gpl_only = false, 1255c4d0bfb4SAlan Maguire .ret_type = RET_INTEGER, 1256c4d0bfb4SAlan Maguire .arg1_type = ARG_PTR_TO_MEM, 1257c4d0bfb4SAlan Maguire .arg2_type = ARG_CONST_SIZE, 1258c4d0bfb4SAlan Maguire .arg3_type = ARG_PTR_TO_MEM, 1259c4d0bfb4SAlan Maguire .arg4_type = ARG_CONST_SIZE, 1260c4d0bfb4SAlan Maguire .arg5_type = ARG_ANYTHING, 1261c4d0bfb4SAlan Maguire }; 1262c4d0bfb4SAlan Maguire 1263fc611f47SKP Singh const struct bpf_func_proto * 1264fc611f47SKP Singh bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 12652541517cSAlexei Starovoitov { 12662541517cSAlexei Starovoitov switch (func_id) { 12672541517cSAlexei Starovoitov case BPF_FUNC_map_lookup_elem: 12682541517cSAlexei Starovoitov return &bpf_map_lookup_elem_proto; 12692541517cSAlexei Starovoitov case BPF_FUNC_map_update_elem: 12702541517cSAlexei Starovoitov return &bpf_map_update_elem_proto; 12712541517cSAlexei Starovoitov case BPF_FUNC_map_delete_elem: 12722541517cSAlexei Starovoitov return &bpf_map_delete_elem_proto; 127302a8c817SAlban Crequy case BPF_FUNC_map_push_elem: 127402a8c817SAlban Crequy return &bpf_map_push_elem_proto; 127502a8c817SAlban Crequy case BPF_FUNC_map_pop_elem: 127602a8c817SAlban Crequy return &bpf_map_pop_elem_proto; 127702a8c817SAlban Crequy case BPF_FUNC_map_peek_elem: 127802a8c817SAlban Crequy return &bpf_map_peek_elem_proto; 1279d9847d31SAlexei Starovoitov case BPF_FUNC_ktime_get_ns: 1280d9847d31SAlexei Starovoitov return &bpf_ktime_get_ns_proto; 128171d19214SMaciej Żenczykowski case BPF_FUNC_ktime_get_boot_ns: 128271d19214SMaciej Żenczykowski return &bpf_ktime_get_boot_ns_proto; 1283d0551261SDmitrii Banshchikov case BPF_FUNC_ktime_get_coarse_ns: 1284d0551261SDmitrii Banshchikov return &bpf_ktime_get_coarse_ns_proto; 128504fd61abSAlexei Starovoitov case BPF_FUNC_tail_call: 128604fd61abSAlexei Starovoitov return &bpf_tail_call_proto; 1287ffeedafbSAlexei Starovoitov case BPF_FUNC_get_current_pid_tgid: 1288ffeedafbSAlexei Starovoitov return &bpf_get_current_pid_tgid_proto; 1289606274c5SAlexei Starovoitov case BPF_FUNC_get_current_task: 1290606274c5SAlexei Starovoitov return &bpf_get_current_task_proto; 12913ca1032aSKP Singh case BPF_FUNC_get_current_task_btf: 12923ca1032aSKP Singh return &bpf_get_current_task_btf_proto; 1293ffeedafbSAlexei Starovoitov case BPF_FUNC_get_current_uid_gid: 1294ffeedafbSAlexei Starovoitov return &bpf_get_current_uid_gid_proto; 1295ffeedafbSAlexei Starovoitov case BPF_FUNC_get_current_comm: 1296ffeedafbSAlexei Starovoitov return &bpf_get_current_comm_proto; 12979c959c86SAlexei Starovoitov case BPF_FUNC_trace_printk: 12980756ea3eSAlexei Starovoitov return bpf_get_trace_printk_proto(); 1299ab1973d3SAlexei Starovoitov case BPF_FUNC_get_smp_processor_id: 1300ab1973d3SAlexei Starovoitov return &bpf_get_smp_processor_id_proto; 13012d0e30c3SDaniel Borkmann case BPF_FUNC_get_numa_node_id: 13022d0e30c3SDaniel Borkmann return &bpf_get_numa_node_id_proto; 130335578d79SKaixu Xia case BPF_FUNC_perf_event_read: 130435578d79SKaixu Xia return &bpf_perf_event_read_proto; 130596ae5227SSargun Dhillon case BPF_FUNC_probe_write_user: 130696ae5227SSargun Dhillon return bpf_get_probe_write_proto(); 130760d20f91SSargun Dhillon case BPF_FUNC_current_task_under_cgroup: 130860d20f91SSargun Dhillon return &bpf_current_task_under_cgroup_proto; 13098937bd80SAlexei Starovoitov case BPF_FUNC_get_prandom_u32: 13108937bd80SAlexei Starovoitov return &bpf_get_prandom_u32_proto; 13116ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read_user: 13126ae08ae3SDaniel Borkmann return &bpf_probe_read_user_proto; 13136ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read_kernel: 13146ae08ae3SDaniel Borkmann return &bpf_probe_read_kernel_proto; 13156ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read_user_str: 13166ae08ae3SDaniel Borkmann return &bpf_probe_read_user_str_proto; 13176ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read_kernel_str: 13186ae08ae3SDaniel Borkmann return &bpf_probe_read_kernel_str_proto; 13190ebeea8cSDaniel Borkmann #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE 13200ebeea8cSDaniel Borkmann case BPF_FUNC_probe_read: 13210ebeea8cSDaniel Borkmann return &bpf_probe_read_compat_proto; 1322a5e8c070SGianluca Borello case BPF_FUNC_probe_read_str: 13236ae08ae3SDaniel Borkmann return &bpf_probe_read_compat_str_proto; 13240ebeea8cSDaniel Borkmann #endif 132534ea38caSYonghong Song #ifdef CONFIG_CGROUPS 1326bf6fa2c8SYonghong Song case BPF_FUNC_get_current_cgroup_id: 1327bf6fa2c8SYonghong Song return &bpf_get_current_cgroup_id_proto; 132834ea38caSYonghong Song #endif 13298b401f9eSYonghong Song case BPF_FUNC_send_signal: 13308b401f9eSYonghong Song return &bpf_send_signal_proto; 13318482941fSYonghong Song case BPF_FUNC_send_signal_thread: 13328482941fSYonghong Song return &bpf_send_signal_thread_proto; 1333b80b033bSSong Liu case BPF_FUNC_perf_event_read_value: 1334b80b033bSSong Liu return &bpf_perf_event_read_value_proto; 1335b4490c5cSCarlos Neira case BPF_FUNC_get_ns_current_pid_tgid: 1336b4490c5cSCarlos Neira return &bpf_get_ns_current_pid_tgid_proto; 1337457f4436SAndrii Nakryiko case BPF_FUNC_ringbuf_output: 1338457f4436SAndrii Nakryiko return &bpf_ringbuf_output_proto; 1339457f4436SAndrii Nakryiko case BPF_FUNC_ringbuf_reserve: 1340457f4436SAndrii Nakryiko return &bpf_ringbuf_reserve_proto; 1341457f4436SAndrii Nakryiko case BPF_FUNC_ringbuf_submit: 1342457f4436SAndrii Nakryiko return &bpf_ringbuf_submit_proto; 1343457f4436SAndrii Nakryiko case BPF_FUNC_ringbuf_discard: 1344457f4436SAndrii Nakryiko return &bpf_ringbuf_discard_proto; 1345457f4436SAndrii Nakryiko case BPF_FUNC_ringbuf_query: 1346457f4436SAndrii Nakryiko return &bpf_ringbuf_query_proto; 134772e2b2b6SYonghong Song case BPF_FUNC_jiffies64: 134872e2b2b6SYonghong Song return &bpf_jiffies64_proto; 1349fa28dcb8SSong Liu case BPF_FUNC_get_task_stack: 1350fa28dcb8SSong Liu return &bpf_get_task_stack_proto; 135107be4c4aSAlexei Starovoitov case BPF_FUNC_copy_from_user: 135207be4c4aSAlexei Starovoitov return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL; 1353c4d0bfb4SAlan Maguire case BPF_FUNC_snprintf_btf: 1354c4d0bfb4SAlan Maguire return &bpf_snprintf_btf_proto; 1355eaa6bcb7SHao Luo case BPF_FUNC_bpf_per_cpu_ptr: 1356eaa6bcb7SHao Luo return &bpf_per_cpu_ptr_proto; 135763d9b80dSHao Luo case BPF_FUNC_bpf_this_cpu_ptr: 135863d9b80dSHao Luo return &bpf_this_cpu_ptr_proto; 13599fd82b61SAlexei Starovoitov default: 13609fd82b61SAlexei Starovoitov return NULL; 13619fd82b61SAlexei Starovoitov } 13629fd82b61SAlexei Starovoitov } 13639fd82b61SAlexei Starovoitov 13645e43f899SAndrey Ignatov static const struct bpf_func_proto * 13655e43f899SAndrey Ignatov kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 13669fd82b61SAlexei Starovoitov { 13679fd82b61SAlexei Starovoitov switch (func_id) { 1368a43eec30SAlexei Starovoitov case BPF_FUNC_perf_event_output: 1369a43eec30SAlexei Starovoitov return &bpf_perf_event_output_proto; 1370d5a3b1f6SAlexei Starovoitov case BPF_FUNC_get_stackid: 1371d5a3b1f6SAlexei Starovoitov return &bpf_get_stackid_proto; 1372c195651eSYonghong Song case BPF_FUNC_get_stack: 1373c195651eSYonghong Song return &bpf_get_stack_proto; 13749802d865SJosef Bacik #ifdef CONFIG_BPF_KPROBE_OVERRIDE 13759802d865SJosef Bacik case BPF_FUNC_override_return: 13769802d865SJosef Bacik return &bpf_override_return_proto; 13779802d865SJosef Bacik #endif 13782541517cSAlexei Starovoitov default: 1379fc611f47SKP Singh return bpf_tracing_func_proto(func_id, prog); 13802541517cSAlexei Starovoitov } 13812541517cSAlexei Starovoitov } 13822541517cSAlexei Starovoitov 13832541517cSAlexei Starovoitov /* bpf+kprobe programs can access fields of 'struct pt_regs' */ 138419de99f7SAlexei Starovoitov static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type type, 13855e43f899SAndrey Ignatov const struct bpf_prog *prog, 138623994631SYonghong Song struct bpf_insn_access_aux *info) 13872541517cSAlexei Starovoitov { 13882541517cSAlexei Starovoitov if (off < 0 || off >= sizeof(struct pt_regs)) 13892541517cSAlexei Starovoitov return false; 13902541517cSAlexei Starovoitov if (type != BPF_READ) 13912541517cSAlexei Starovoitov return false; 13922541517cSAlexei Starovoitov if (off % size != 0) 13932541517cSAlexei Starovoitov return false; 13942d071c64SDaniel Borkmann /* 13952d071c64SDaniel Borkmann * Assertion for 32 bit to make sure last 8 byte access 13962d071c64SDaniel Borkmann * (BPF_DW) to the last 4 byte member is disallowed. 13972d071c64SDaniel Borkmann */ 13982d071c64SDaniel Borkmann if (off + size > sizeof(struct pt_regs)) 13992d071c64SDaniel Borkmann return false; 14002d071c64SDaniel Borkmann 14012541517cSAlexei Starovoitov return true; 14022541517cSAlexei Starovoitov } 14032541517cSAlexei Starovoitov 14047de16e3aSJakub Kicinski const struct bpf_verifier_ops kprobe_verifier_ops = { 14052541517cSAlexei Starovoitov .get_func_proto = kprobe_prog_func_proto, 14062541517cSAlexei Starovoitov .is_valid_access = kprobe_prog_is_valid_access, 14072541517cSAlexei Starovoitov }; 14082541517cSAlexei Starovoitov 14097de16e3aSJakub Kicinski const struct bpf_prog_ops kprobe_prog_ops = { 14107de16e3aSJakub Kicinski }; 14117de16e3aSJakub Kicinski 1412f3694e00SDaniel Borkmann BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map, 1413f3694e00SDaniel Borkmann u64, flags, void *, data, u64, size) 14149940d67cSAlexei Starovoitov { 1415f3694e00SDaniel Borkmann struct pt_regs *regs = *(struct pt_regs **)tp_buff; 1416f3694e00SDaniel Borkmann 14179940d67cSAlexei Starovoitov /* 14189940d67cSAlexei Starovoitov * r1 points to perf tracepoint buffer where first 8 bytes are hidden 14199940d67cSAlexei Starovoitov * from bpf program and contain a pointer to 'struct pt_regs'. Fetch it 1420f3694e00SDaniel Borkmann * from there and call the same bpf_perf_event_output() helper inline. 14219940d67cSAlexei Starovoitov */ 1422f3694e00SDaniel Borkmann return ____bpf_perf_event_output(regs, map, flags, data, size); 14239940d67cSAlexei Starovoitov } 14249940d67cSAlexei Starovoitov 14259940d67cSAlexei Starovoitov static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { 14269940d67cSAlexei Starovoitov .func = bpf_perf_event_output_tp, 14279940d67cSAlexei Starovoitov .gpl_only = true, 14289940d67cSAlexei Starovoitov .ret_type = RET_INTEGER, 14299940d67cSAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 14309940d67cSAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 14319940d67cSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 143239f19ebbSAlexei Starovoitov .arg4_type = ARG_PTR_TO_MEM, 1433a60dd35dSGianluca Borello .arg5_type = ARG_CONST_SIZE_OR_ZERO, 14349940d67cSAlexei Starovoitov }; 14359940d67cSAlexei Starovoitov 1436f3694e00SDaniel Borkmann BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map, 1437f3694e00SDaniel Borkmann u64, flags) 14389940d67cSAlexei Starovoitov { 1439f3694e00SDaniel Borkmann struct pt_regs *regs = *(struct pt_regs **)tp_buff; 14409940d67cSAlexei Starovoitov 1441f3694e00SDaniel Borkmann /* 1442f3694e00SDaniel Borkmann * Same comment as in bpf_perf_event_output_tp(), only that this time 1443f3694e00SDaniel Borkmann * the other helper's function body cannot be inlined due to being 1444f3694e00SDaniel Borkmann * external, thus we need to call raw helper function. 1445f3694e00SDaniel Borkmann */ 1446f3694e00SDaniel Borkmann return bpf_get_stackid((unsigned long) regs, (unsigned long) map, 1447f3694e00SDaniel Borkmann flags, 0, 0); 14489940d67cSAlexei Starovoitov } 14499940d67cSAlexei Starovoitov 14509940d67cSAlexei Starovoitov static const struct bpf_func_proto bpf_get_stackid_proto_tp = { 14519940d67cSAlexei Starovoitov .func = bpf_get_stackid_tp, 14529940d67cSAlexei Starovoitov .gpl_only = true, 14539940d67cSAlexei Starovoitov .ret_type = RET_INTEGER, 14549940d67cSAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 14559940d67cSAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 14569940d67cSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 14579940d67cSAlexei Starovoitov }; 14589940d67cSAlexei Starovoitov 1459c195651eSYonghong Song BPF_CALL_4(bpf_get_stack_tp, void *, tp_buff, void *, buf, u32, size, 1460c195651eSYonghong Song u64, flags) 1461c195651eSYonghong Song { 1462c195651eSYonghong Song struct pt_regs *regs = *(struct pt_regs **)tp_buff; 1463c195651eSYonghong Song 1464c195651eSYonghong Song return bpf_get_stack((unsigned long) regs, (unsigned long) buf, 1465c195651eSYonghong Song (unsigned long) size, flags, 0); 1466c195651eSYonghong Song } 1467c195651eSYonghong Song 1468c195651eSYonghong Song static const struct bpf_func_proto bpf_get_stack_proto_tp = { 1469c195651eSYonghong Song .func = bpf_get_stack_tp, 1470c195651eSYonghong Song .gpl_only = true, 1471c195651eSYonghong Song .ret_type = RET_INTEGER, 1472c195651eSYonghong Song .arg1_type = ARG_PTR_TO_CTX, 1473c195651eSYonghong Song .arg2_type = ARG_PTR_TO_UNINIT_MEM, 1474c195651eSYonghong Song .arg3_type = ARG_CONST_SIZE_OR_ZERO, 1475c195651eSYonghong Song .arg4_type = ARG_ANYTHING, 1476c195651eSYonghong Song }; 1477c195651eSYonghong Song 14785e43f899SAndrey Ignatov static const struct bpf_func_proto * 14795e43f899SAndrey Ignatov tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 14809fd82b61SAlexei Starovoitov { 14819fd82b61SAlexei Starovoitov switch (func_id) { 14829fd82b61SAlexei Starovoitov case BPF_FUNC_perf_event_output: 14839940d67cSAlexei Starovoitov return &bpf_perf_event_output_proto_tp; 14849fd82b61SAlexei Starovoitov case BPF_FUNC_get_stackid: 14859940d67cSAlexei Starovoitov return &bpf_get_stackid_proto_tp; 1486c195651eSYonghong Song case BPF_FUNC_get_stack: 1487c195651eSYonghong Song return &bpf_get_stack_proto_tp; 14889fd82b61SAlexei Starovoitov default: 1489fc611f47SKP Singh return bpf_tracing_func_proto(func_id, prog); 14909fd82b61SAlexei Starovoitov } 14919fd82b61SAlexei Starovoitov } 14929fd82b61SAlexei Starovoitov 149319de99f7SAlexei Starovoitov static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type, 14945e43f899SAndrey Ignatov const struct bpf_prog *prog, 149523994631SYonghong Song struct bpf_insn_access_aux *info) 14969fd82b61SAlexei Starovoitov { 14979fd82b61SAlexei Starovoitov if (off < sizeof(void *) || off >= PERF_MAX_TRACE_SIZE) 14989fd82b61SAlexei Starovoitov return false; 14999fd82b61SAlexei Starovoitov if (type != BPF_READ) 15009fd82b61SAlexei Starovoitov return false; 15019fd82b61SAlexei Starovoitov if (off % size != 0) 15029fd82b61SAlexei Starovoitov return false; 15032d071c64SDaniel Borkmann 15042d071c64SDaniel Borkmann BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(__u64)); 15059fd82b61SAlexei Starovoitov return true; 15069fd82b61SAlexei Starovoitov } 15079fd82b61SAlexei Starovoitov 15087de16e3aSJakub Kicinski const struct bpf_verifier_ops tracepoint_verifier_ops = { 15099fd82b61SAlexei Starovoitov .get_func_proto = tp_prog_func_proto, 15109fd82b61SAlexei Starovoitov .is_valid_access = tp_prog_is_valid_access, 15119fd82b61SAlexei Starovoitov }; 15129fd82b61SAlexei Starovoitov 15137de16e3aSJakub Kicinski const struct bpf_prog_ops tracepoint_prog_ops = { 15147de16e3aSJakub Kicinski }; 15157de16e3aSJakub Kicinski 1516f005afedSYonghong Song BPF_CALL_3(bpf_perf_prog_read_value, struct bpf_perf_event_data_kern *, ctx, 1517f005afedSYonghong Song struct bpf_perf_event_value *, buf, u32, size) 1518f005afedSYonghong Song { 1519f005afedSYonghong Song int err = -EINVAL; 1520f005afedSYonghong Song 1521f005afedSYonghong Song if (unlikely(size != sizeof(struct bpf_perf_event_value))) 1522f005afedSYonghong Song goto clear; 1523f005afedSYonghong Song err = perf_event_read_local(ctx->event, &buf->counter, &buf->enabled, 1524f005afedSYonghong Song &buf->running); 1525f005afedSYonghong Song if (unlikely(err)) 1526f005afedSYonghong Song goto clear; 1527f005afedSYonghong Song return 0; 1528f005afedSYonghong Song clear: 1529f005afedSYonghong Song memset(buf, 0, size); 1530f005afedSYonghong Song return err; 1531f005afedSYonghong Song } 1532f005afedSYonghong Song 1533f005afedSYonghong Song static const struct bpf_func_proto bpf_perf_prog_read_value_proto = { 1534f005afedSYonghong Song .func = bpf_perf_prog_read_value, 1535f005afedSYonghong Song .gpl_only = true, 1536f005afedSYonghong Song .ret_type = RET_INTEGER, 1537f005afedSYonghong Song .arg1_type = ARG_PTR_TO_CTX, 1538f005afedSYonghong Song .arg2_type = ARG_PTR_TO_UNINIT_MEM, 1539f005afedSYonghong Song .arg3_type = ARG_CONST_SIZE, 1540f005afedSYonghong Song }; 1541f005afedSYonghong Song 1542fff7b643SDaniel Xu BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx, 1543fff7b643SDaniel Xu void *, buf, u32, size, u64, flags) 1544fff7b643SDaniel Xu { 1545fff7b643SDaniel Xu #ifndef CONFIG_X86 1546fff7b643SDaniel Xu return -ENOENT; 1547fff7b643SDaniel Xu #else 1548fff7b643SDaniel Xu static const u32 br_entry_size = sizeof(struct perf_branch_entry); 1549fff7b643SDaniel Xu struct perf_branch_stack *br_stack = ctx->data->br_stack; 1550fff7b643SDaniel Xu u32 to_copy; 1551fff7b643SDaniel Xu 1552fff7b643SDaniel Xu if (unlikely(flags & ~BPF_F_GET_BRANCH_RECORDS_SIZE)) 1553fff7b643SDaniel Xu return -EINVAL; 1554fff7b643SDaniel Xu 1555fff7b643SDaniel Xu if (unlikely(!br_stack)) 1556fff7b643SDaniel Xu return -EINVAL; 1557fff7b643SDaniel Xu 1558fff7b643SDaniel Xu if (flags & BPF_F_GET_BRANCH_RECORDS_SIZE) 1559fff7b643SDaniel Xu return br_stack->nr * br_entry_size; 1560fff7b643SDaniel Xu 1561fff7b643SDaniel Xu if (!buf || (size % br_entry_size != 0)) 1562fff7b643SDaniel Xu return -EINVAL; 1563fff7b643SDaniel Xu 1564fff7b643SDaniel Xu to_copy = min_t(u32, br_stack->nr * br_entry_size, size); 1565fff7b643SDaniel Xu memcpy(buf, br_stack->entries, to_copy); 1566fff7b643SDaniel Xu 1567fff7b643SDaniel Xu return to_copy; 1568fff7b643SDaniel Xu #endif 1569fff7b643SDaniel Xu } 1570fff7b643SDaniel Xu 1571fff7b643SDaniel Xu static const struct bpf_func_proto bpf_read_branch_records_proto = { 1572fff7b643SDaniel Xu .func = bpf_read_branch_records, 1573fff7b643SDaniel Xu .gpl_only = true, 1574fff7b643SDaniel Xu .ret_type = RET_INTEGER, 1575fff7b643SDaniel Xu .arg1_type = ARG_PTR_TO_CTX, 1576fff7b643SDaniel Xu .arg2_type = ARG_PTR_TO_MEM_OR_NULL, 1577fff7b643SDaniel Xu .arg3_type = ARG_CONST_SIZE_OR_ZERO, 1578fff7b643SDaniel Xu .arg4_type = ARG_ANYTHING, 1579fff7b643SDaniel Xu }; 1580fff7b643SDaniel Xu 15815e43f899SAndrey Ignatov static const struct bpf_func_proto * 15825e43f899SAndrey Ignatov pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1583f005afedSYonghong Song { 1584f005afedSYonghong Song switch (func_id) { 1585f005afedSYonghong Song case BPF_FUNC_perf_event_output: 1586f005afedSYonghong Song return &bpf_perf_event_output_proto_tp; 1587f005afedSYonghong Song case BPF_FUNC_get_stackid: 15887b04d6d6SSong Liu return &bpf_get_stackid_proto_pe; 1589c195651eSYonghong Song case BPF_FUNC_get_stack: 15907b04d6d6SSong Liu return &bpf_get_stack_proto_pe; 1591f005afedSYonghong Song case BPF_FUNC_perf_prog_read_value: 1592f005afedSYonghong Song return &bpf_perf_prog_read_value_proto; 1593fff7b643SDaniel Xu case BPF_FUNC_read_branch_records: 1594fff7b643SDaniel Xu return &bpf_read_branch_records_proto; 1595f005afedSYonghong Song default: 1596fc611f47SKP Singh return bpf_tracing_func_proto(func_id, prog); 1597f005afedSYonghong Song } 1598f005afedSYonghong Song } 1599f005afedSYonghong Song 1600c4f6699dSAlexei Starovoitov /* 1601c4f6699dSAlexei Starovoitov * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp 1602c4f6699dSAlexei Starovoitov * to avoid potential recursive reuse issue when/if tracepoints are added 16039594dc3cSMatt Mullins * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack. 16049594dc3cSMatt Mullins * 16059594dc3cSMatt Mullins * Since raw tracepoints run despite bpf_prog_active, support concurrent usage 16069594dc3cSMatt Mullins * in normal, irq, and nmi context. 1607c4f6699dSAlexei Starovoitov */ 16089594dc3cSMatt Mullins struct bpf_raw_tp_regs { 16099594dc3cSMatt Mullins struct pt_regs regs[3]; 16109594dc3cSMatt Mullins }; 16119594dc3cSMatt Mullins static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs); 16129594dc3cSMatt Mullins static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level); 16139594dc3cSMatt Mullins static struct pt_regs *get_bpf_raw_tp_regs(void) 16149594dc3cSMatt Mullins { 16159594dc3cSMatt Mullins struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs); 16169594dc3cSMatt Mullins int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level); 16179594dc3cSMatt Mullins 16189594dc3cSMatt Mullins if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(tp_regs->regs))) { 16199594dc3cSMatt Mullins this_cpu_dec(bpf_raw_tp_nest_level); 16209594dc3cSMatt Mullins return ERR_PTR(-EBUSY); 16219594dc3cSMatt Mullins } 16229594dc3cSMatt Mullins 16239594dc3cSMatt Mullins return &tp_regs->regs[nest_level - 1]; 16249594dc3cSMatt Mullins } 16259594dc3cSMatt Mullins 16269594dc3cSMatt Mullins static void put_bpf_raw_tp_regs(void) 16279594dc3cSMatt Mullins { 16289594dc3cSMatt Mullins this_cpu_dec(bpf_raw_tp_nest_level); 16299594dc3cSMatt Mullins } 16309594dc3cSMatt Mullins 1631c4f6699dSAlexei Starovoitov BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args, 1632c4f6699dSAlexei Starovoitov struct bpf_map *, map, u64, flags, void *, data, u64, size) 1633c4f6699dSAlexei Starovoitov { 16349594dc3cSMatt Mullins struct pt_regs *regs = get_bpf_raw_tp_regs(); 16359594dc3cSMatt Mullins int ret; 16369594dc3cSMatt Mullins 16379594dc3cSMatt Mullins if (IS_ERR(regs)) 16389594dc3cSMatt Mullins return PTR_ERR(regs); 1639c4f6699dSAlexei Starovoitov 1640c4f6699dSAlexei Starovoitov perf_fetch_caller_regs(regs); 16419594dc3cSMatt Mullins ret = ____bpf_perf_event_output(regs, map, flags, data, size); 16429594dc3cSMatt Mullins 16439594dc3cSMatt Mullins put_bpf_raw_tp_regs(); 16449594dc3cSMatt Mullins return ret; 1645c4f6699dSAlexei Starovoitov } 1646c4f6699dSAlexei Starovoitov 1647c4f6699dSAlexei Starovoitov static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { 1648c4f6699dSAlexei Starovoitov .func = bpf_perf_event_output_raw_tp, 1649c4f6699dSAlexei Starovoitov .gpl_only = true, 1650c4f6699dSAlexei Starovoitov .ret_type = RET_INTEGER, 1651c4f6699dSAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 1652c4f6699dSAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 1653c4f6699dSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 1654c4f6699dSAlexei Starovoitov .arg4_type = ARG_PTR_TO_MEM, 1655c4f6699dSAlexei Starovoitov .arg5_type = ARG_CONST_SIZE_OR_ZERO, 1656c4f6699dSAlexei Starovoitov }; 1657c4f6699dSAlexei Starovoitov 1658a7658e1aSAlexei Starovoitov extern const struct bpf_func_proto bpf_skb_output_proto; 1659d831ee84SEelco Chaudron extern const struct bpf_func_proto bpf_xdp_output_proto; 1660a7658e1aSAlexei Starovoitov 1661c4f6699dSAlexei Starovoitov BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args, 1662c4f6699dSAlexei Starovoitov struct bpf_map *, map, u64, flags) 1663c4f6699dSAlexei Starovoitov { 16649594dc3cSMatt Mullins struct pt_regs *regs = get_bpf_raw_tp_regs(); 16659594dc3cSMatt Mullins int ret; 16669594dc3cSMatt Mullins 16679594dc3cSMatt Mullins if (IS_ERR(regs)) 16689594dc3cSMatt Mullins return PTR_ERR(regs); 1669c4f6699dSAlexei Starovoitov 1670c4f6699dSAlexei Starovoitov perf_fetch_caller_regs(regs); 1671c4f6699dSAlexei Starovoitov /* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */ 16729594dc3cSMatt Mullins ret = bpf_get_stackid((unsigned long) regs, (unsigned long) map, 1673c4f6699dSAlexei Starovoitov flags, 0, 0); 16749594dc3cSMatt Mullins put_bpf_raw_tp_regs(); 16759594dc3cSMatt Mullins return ret; 1676c4f6699dSAlexei Starovoitov } 1677c4f6699dSAlexei Starovoitov 1678c4f6699dSAlexei Starovoitov static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = { 1679c4f6699dSAlexei Starovoitov .func = bpf_get_stackid_raw_tp, 1680c4f6699dSAlexei Starovoitov .gpl_only = true, 1681c4f6699dSAlexei Starovoitov .ret_type = RET_INTEGER, 1682c4f6699dSAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 1683c4f6699dSAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 1684c4f6699dSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 1685c4f6699dSAlexei Starovoitov }; 1686c4f6699dSAlexei Starovoitov 1687c195651eSYonghong Song BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args, 1688c195651eSYonghong Song void *, buf, u32, size, u64, flags) 1689c195651eSYonghong Song { 16909594dc3cSMatt Mullins struct pt_regs *regs = get_bpf_raw_tp_regs(); 16919594dc3cSMatt Mullins int ret; 16929594dc3cSMatt Mullins 16939594dc3cSMatt Mullins if (IS_ERR(regs)) 16949594dc3cSMatt Mullins return PTR_ERR(regs); 1695c195651eSYonghong Song 1696c195651eSYonghong Song perf_fetch_caller_regs(regs); 16979594dc3cSMatt Mullins ret = bpf_get_stack((unsigned long) regs, (unsigned long) buf, 1698c195651eSYonghong Song (unsigned long) size, flags, 0); 16999594dc3cSMatt Mullins put_bpf_raw_tp_regs(); 17009594dc3cSMatt Mullins return ret; 1701c195651eSYonghong Song } 1702c195651eSYonghong Song 1703c195651eSYonghong Song static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { 1704c195651eSYonghong Song .func = bpf_get_stack_raw_tp, 1705c195651eSYonghong Song .gpl_only = true, 1706c195651eSYonghong Song .ret_type = RET_INTEGER, 1707c195651eSYonghong Song .arg1_type = ARG_PTR_TO_CTX, 1708c195651eSYonghong Song .arg2_type = ARG_PTR_TO_MEM, 1709c195651eSYonghong Song .arg3_type = ARG_CONST_SIZE_OR_ZERO, 1710c195651eSYonghong Song .arg4_type = ARG_ANYTHING, 1711c195651eSYonghong Song }; 1712c195651eSYonghong Song 17135e43f899SAndrey Ignatov static const struct bpf_func_proto * 17145e43f899SAndrey Ignatov raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1715c4f6699dSAlexei Starovoitov { 1716c4f6699dSAlexei Starovoitov switch (func_id) { 1717c4f6699dSAlexei Starovoitov case BPF_FUNC_perf_event_output: 1718c4f6699dSAlexei Starovoitov return &bpf_perf_event_output_proto_raw_tp; 1719c4f6699dSAlexei Starovoitov case BPF_FUNC_get_stackid: 1720c4f6699dSAlexei Starovoitov return &bpf_get_stackid_proto_raw_tp; 1721c195651eSYonghong Song case BPF_FUNC_get_stack: 1722c195651eSYonghong Song return &bpf_get_stack_proto_raw_tp; 1723c4f6699dSAlexei Starovoitov default: 1724fc611f47SKP Singh return bpf_tracing_func_proto(func_id, prog); 1725c4f6699dSAlexei Starovoitov } 1726c4f6699dSAlexei Starovoitov } 1727c4f6699dSAlexei Starovoitov 1728958a3f2dSJiri Olsa const struct bpf_func_proto * 1729f1b9509cSAlexei Starovoitov tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1730f1b9509cSAlexei Starovoitov { 1731f1b9509cSAlexei Starovoitov switch (func_id) { 1732f1b9509cSAlexei Starovoitov #ifdef CONFIG_NET 1733f1b9509cSAlexei Starovoitov case BPF_FUNC_skb_output: 1734f1b9509cSAlexei Starovoitov return &bpf_skb_output_proto; 1735d831ee84SEelco Chaudron case BPF_FUNC_xdp_output: 1736d831ee84SEelco Chaudron return &bpf_xdp_output_proto; 1737af7ec138SYonghong Song case BPF_FUNC_skc_to_tcp6_sock: 1738af7ec138SYonghong Song return &bpf_skc_to_tcp6_sock_proto; 1739478cfbdfSYonghong Song case BPF_FUNC_skc_to_tcp_sock: 1740478cfbdfSYonghong Song return &bpf_skc_to_tcp_sock_proto; 1741478cfbdfSYonghong Song case BPF_FUNC_skc_to_tcp_timewait_sock: 1742478cfbdfSYonghong Song return &bpf_skc_to_tcp_timewait_sock_proto; 1743478cfbdfSYonghong Song case BPF_FUNC_skc_to_tcp_request_sock: 1744478cfbdfSYonghong Song return &bpf_skc_to_tcp_request_sock_proto; 17450d4fad3eSYonghong Song case BPF_FUNC_skc_to_udp6_sock: 17460d4fad3eSYonghong Song return &bpf_skc_to_udp6_sock_proto; 17478e4597c6SMartin KaFai Lau case BPF_FUNC_sk_storage_get: 17488e4597c6SMartin KaFai Lau return &bpf_sk_storage_get_tracing_proto; 17498e4597c6SMartin KaFai Lau case BPF_FUNC_sk_storage_delete: 17508e4597c6SMartin KaFai Lau return &bpf_sk_storage_delete_tracing_proto; 1751f1b9509cSAlexei Starovoitov #endif 1752492e639fSYonghong Song case BPF_FUNC_seq_printf: 1753492e639fSYonghong Song return prog->expected_attach_type == BPF_TRACE_ITER ? 1754492e639fSYonghong Song &bpf_seq_printf_proto : 1755492e639fSYonghong Song NULL; 1756492e639fSYonghong Song case BPF_FUNC_seq_write: 1757492e639fSYonghong Song return prog->expected_attach_type == BPF_TRACE_ITER ? 1758492e639fSYonghong Song &bpf_seq_write_proto : 1759492e639fSYonghong Song NULL; 1760eb411377SAlan Maguire case BPF_FUNC_seq_printf_btf: 1761eb411377SAlan Maguire return prog->expected_attach_type == BPF_TRACE_ITER ? 1762eb411377SAlan Maguire &bpf_seq_printf_btf_proto : 1763eb411377SAlan Maguire NULL; 17646e22ab9dSJiri Olsa case BPF_FUNC_d_path: 17656e22ab9dSJiri Olsa return &bpf_d_path_proto; 1766f1b9509cSAlexei Starovoitov default: 1767f1b9509cSAlexei Starovoitov return raw_tp_prog_func_proto(func_id, prog); 1768f1b9509cSAlexei Starovoitov } 1769f1b9509cSAlexei Starovoitov } 1770f1b9509cSAlexei Starovoitov 1771c4f6699dSAlexei Starovoitov static bool raw_tp_prog_is_valid_access(int off, int size, 1772c4f6699dSAlexei Starovoitov enum bpf_access_type type, 17735e43f899SAndrey Ignatov const struct bpf_prog *prog, 1774c4f6699dSAlexei Starovoitov struct bpf_insn_access_aux *info) 1775c4f6699dSAlexei Starovoitov { 1776f1b9509cSAlexei Starovoitov if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 1777c4f6699dSAlexei Starovoitov return false; 1778c4f6699dSAlexei Starovoitov if (type != BPF_READ) 1779c4f6699dSAlexei Starovoitov return false; 1780c4f6699dSAlexei Starovoitov if (off % size != 0) 1781c4f6699dSAlexei Starovoitov return false; 178238207291SMartin KaFai Lau return true; 1783f1b9509cSAlexei Starovoitov } 1784f1b9509cSAlexei Starovoitov 1785f1b9509cSAlexei Starovoitov static bool tracing_prog_is_valid_access(int off, int size, 1786f1b9509cSAlexei Starovoitov enum bpf_access_type type, 1787f1b9509cSAlexei Starovoitov const struct bpf_prog *prog, 1788f1b9509cSAlexei Starovoitov struct bpf_insn_access_aux *info) 1789f1b9509cSAlexei Starovoitov { 1790f1b9509cSAlexei Starovoitov if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 1791f1b9509cSAlexei Starovoitov return false; 1792f1b9509cSAlexei Starovoitov if (type != BPF_READ) 1793f1b9509cSAlexei Starovoitov return false; 1794f1b9509cSAlexei Starovoitov if (off % size != 0) 1795f1b9509cSAlexei Starovoitov return false; 17969e15db66SAlexei Starovoitov return btf_ctx_access(off, size, type, prog, info); 1797c4f6699dSAlexei Starovoitov } 1798c4f6699dSAlexei Starovoitov 17993e7c67d9SKP Singh int __weak bpf_prog_test_run_tracing(struct bpf_prog *prog, 18003e7c67d9SKP Singh const union bpf_attr *kattr, 18013e7c67d9SKP Singh union bpf_attr __user *uattr) 18023e7c67d9SKP Singh { 18033e7c67d9SKP Singh return -ENOTSUPP; 18043e7c67d9SKP Singh } 18053e7c67d9SKP Singh 1806c4f6699dSAlexei Starovoitov const struct bpf_verifier_ops raw_tracepoint_verifier_ops = { 1807c4f6699dSAlexei Starovoitov .get_func_proto = raw_tp_prog_func_proto, 1808c4f6699dSAlexei Starovoitov .is_valid_access = raw_tp_prog_is_valid_access, 1809c4f6699dSAlexei Starovoitov }; 1810c4f6699dSAlexei Starovoitov 1811c4f6699dSAlexei Starovoitov const struct bpf_prog_ops raw_tracepoint_prog_ops = { 1812ebfb4d40SYonghong Song #ifdef CONFIG_NET 18131b4d60ecSSong Liu .test_run = bpf_prog_test_run_raw_tp, 1814ebfb4d40SYonghong Song #endif 1815c4f6699dSAlexei Starovoitov }; 1816c4f6699dSAlexei Starovoitov 1817f1b9509cSAlexei Starovoitov const struct bpf_verifier_ops tracing_verifier_ops = { 1818f1b9509cSAlexei Starovoitov .get_func_proto = tracing_prog_func_proto, 1819f1b9509cSAlexei Starovoitov .is_valid_access = tracing_prog_is_valid_access, 1820f1b9509cSAlexei Starovoitov }; 1821f1b9509cSAlexei Starovoitov 1822f1b9509cSAlexei Starovoitov const struct bpf_prog_ops tracing_prog_ops = { 1823da00d2f1SKP Singh .test_run = bpf_prog_test_run_tracing, 1824f1b9509cSAlexei Starovoitov }; 1825f1b9509cSAlexei Starovoitov 18269df1c28bSMatt Mullins static bool raw_tp_writable_prog_is_valid_access(int off, int size, 18279df1c28bSMatt Mullins enum bpf_access_type type, 18289df1c28bSMatt Mullins const struct bpf_prog *prog, 18299df1c28bSMatt Mullins struct bpf_insn_access_aux *info) 18309df1c28bSMatt Mullins { 18319df1c28bSMatt Mullins if (off == 0) { 18329df1c28bSMatt Mullins if (size != sizeof(u64) || type != BPF_READ) 18339df1c28bSMatt Mullins return false; 18349df1c28bSMatt Mullins info->reg_type = PTR_TO_TP_BUFFER; 18359df1c28bSMatt Mullins } 18369df1c28bSMatt Mullins return raw_tp_prog_is_valid_access(off, size, type, prog, info); 18379df1c28bSMatt Mullins } 18389df1c28bSMatt Mullins 18399df1c28bSMatt Mullins const struct bpf_verifier_ops raw_tracepoint_writable_verifier_ops = { 18409df1c28bSMatt Mullins .get_func_proto = raw_tp_prog_func_proto, 18419df1c28bSMatt Mullins .is_valid_access = raw_tp_writable_prog_is_valid_access, 18429df1c28bSMatt Mullins }; 18439df1c28bSMatt Mullins 18449df1c28bSMatt Mullins const struct bpf_prog_ops raw_tracepoint_writable_prog_ops = { 18459df1c28bSMatt Mullins }; 18469df1c28bSMatt Mullins 18470515e599SAlexei Starovoitov static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type, 18485e43f899SAndrey Ignatov const struct bpf_prog *prog, 184923994631SYonghong Song struct bpf_insn_access_aux *info) 18500515e599SAlexei Starovoitov { 185195da0cdbSTeng Qin const int size_u64 = sizeof(u64); 185231fd8581SYonghong Song 18530515e599SAlexei Starovoitov if (off < 0 || off >= sizeof(struct bpf_perf_event_data)) 18540515e599SAlexei Starovoitov return false; 18550515e599SAlexei Starovoitov if (type != BPF_READ) 18560515e599SAlexei Starovoitov return false; 1857bc23105cSDaniel Borkmann if (off % size != 0) { 1858bc23105cSDaniel Borkmann if (sizeof(unsigned long) != 4) 18590515e599SAlexei Starovoitov return false; 1860bc23105cSDaniel Borkmann if (size != 8) 1861bc23105cSDaniel Borkmann return false; 1862bc23105cSDaniel Borkmann if (off % size != 4) 1863bc23105cSDaniel Borkmann return false; 1864bc23105cSDaniel Borkmann } 186531fd8581SYonghong Song 1866f96da094SDaniel Borkmann switch (off) { 1867f96da094SDaniel Borkmann case bpf_ctx_range(struct bpf_perf_event_data, sample_period): 186895da0cdbSTeng Qin bpf_ctx_record_field_size(info, size_u64); 186995da0cdbSTeng Qin if (!bpf_ctx_narrow_access_ok(off, size, size_u64)) 187095da0cdbSTeng Qin return false; 187195da0cdbSTeng Qin break; 187295da0cdbSTeng Qin case bpf_ctx_range(struct bpf_perf_event_data, addr): 187395da0cdbSTeng Qin bpf_ctx_record_field_size(info, size_u64); 187495da0cdbSTeng Qin if (!bpf_ctx_narrow_access_ok(off, size, size_u64)) 187523994631SYonghong Song return false; 1876f96da094SDaniel Borkmann break; 1877f96da094SDaniel Borkmann default: 18780515e599SAlexei Starovoitov if (size != sizeof(long)) 18790515e599SAlexei Starovoitov return false; 18800515e599SAlexei Starovoitov } 1881f96da094SDaniel Borkmann 18820515e599SAlexei Starovoitov return true; 18830515e599SAlexei Starovoitov } 18840515e599SAlexei Starovoitov 18856b8cc1d1SDaniel Borkmann static u32 pe_prog_convert_ctx_access(enum bpf_access_type type, 18866b8cc1d1SDaniel Borkmann const struct bpf_insn *si, 18870515e599SAlexei Starovoitov struct bpf_insn *insn_buf, 1888f96da094SDaniel Borkmann struct bpf_prog *prog, u32 *target_size) 18890515e599SAlexei Starovoitov { 18900515e599SAlexei Starovoitov struct bpf_insn *insn = insn_buf; 18910515e599SAlexei Starovoitov 18926b8cc1d1SDaniel Borkmann switch (si->off) { 18930515e599SAlexei Starovoitov case offsetof(struct bpf_perf_event_data, sample_period): 1894f035a515SDaniel Borkmann *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 18956b8cc1d1SDaniel Borkmann data), si->dst_reg, si->src_reg, 18960515e599SAlexei Starovoitov offsetof(struct bpf_perf_event_data_kern, data)); 18976b8cc1d1SDaniel Borkmann *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, 1898f96da094SDaniel Borkmann bpf_target_off(struct perf_sample_data, period, 8, 1899f96da094SDaniel Borkmann target_size)); 19000515e599SAlexei Starovoitov break; 190195da0cdbSTeng Qin case offsetof(struct bpf_perf_event_data, addr): 190295da0cdbSTeng Qin *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 190395da0cdbSTeng Qin data), si->dst_reg, si->src_reg, 190495da0cdbSTeng Qin offsetof(struct bpf_perf_event_data_kern, data)); 190595da0cdbSTeng Qin *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, 190695da0cdbSTeng Qin bpf_target_off(struct perf_sample_data, addr, 8, 190795da0cdbSTeng Qin target_size)); 190895da0cdbSTeng Qin break; 19090515e599SAlexei Starovoitov default: 1910f035a515SDaniel Borkmann *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 19116b8cc1d1SDaniel Borkmann regs), si->dst_reg, si->src_reg, 19120515e599SAlexei Starovoitov offsetof(struct bpf_perf_event_data_kern, regs)); 19136b8cc1d1SDaniel Borkmann *insn++ = BPF_LDX_MEM(BPF_SIZEOF(long), si->dst_reg, si->dst_reg, 19146b8cc1d1SDaniel Borkmann si->off); 19150515e599SAlexei Starovoitov break; 19160515e599SAlexei Starovoitov } 19170515e599SAlexei Starovoitov 19180515e599SAlexei Starovoitov return insn - insn_buf; 19190515e599SAlexei Starovoitov } 19200515e599SAlexei Starovoitov 19217de16e3aSJakub Kicinski const struct bpf_verifier_ops perf_event_verifier_ops = { 1922f005afedSYonghong Song .get_func_proto = pe_prog_func_proto, 19230515e599SAlexei Starovoitov .is_valid_access = pe_prog_is_valid_access, 19240515e599SAlexei Starovoitov .convert_ctx_access = pe_prog_convert_ctx_access, 19250515e599SAlexei Starovoitov }; 19267de16e3aSJakub Kicinski 19277de16e3aSJakub Kicinski const struct bpf_prog_ops perf_event_prog_ops = { 19287de16e3aSJakub Kicinski }; 1929e87c6bc3SYonghong Song 1930e87c6bc3SYonghong Song static DEFINE_MUTEX(bpf_event_mutex); 1931e87c6bc3SYonghong Song 1932c8c088baSYonghong Song #define BPF_TRACE_MAX_PROGS 64 1933c8c088baSYonghong Song 1934e87c6bc3SYonghong Song int perf_event_attach_bpf_prog(struct perf_event *event, 1935e87c6bc3SYonghong Song struct bpf_prog *prog) 1936e87c6bc3SYonghong Song { 1937e672db03SStanislav Fomichev struct bpf_prog_array *old_array; 1938e87c6bc3SYonghong Song struct bpf_prog_array *new_array; 1939e87c6bc3SYonghong Song int ret = -EEXIST; 1940e87c6bc3SYonghong Song 19419802d865SJosef Bacik /* 1942b4da3340SMasami Hiramatsu * Kprobe override only works if they are on the function entry, 1943b4da3340SMasami Hiramatsu * and only if they are on the opt-in list. 19449802d865SJosef Bacik */ 19459802d865SJosef Bacik if (prog->kprobe_override && 1946b4da3340SMasami Hiramatsu (!trace_kprobe_on_func_entry(event->tp_event) || 19479802d865SJosef Bacik !trace_kprobe_error_injectable(event->tp_event))) 19489802d865SJosef Bacik return -EINVAL; 19499802d865SJosef Bacik 1950e87c6bc3SYonghong Song mutex_lock(&bpf_event_mutex); 1951e87c6bc3SYonghong Song 1952e87c6bc3SYonghong Song if (event->prog) 195307c41a29SYonghong Song goto unlock; 1954e87c6bc3SYonghong Song 1955e672db03SStanislav Fomichev old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); 1956c8c088baSYonghong Song if (old_array && 1957c8c088baSYonghong Song bpf_prog_array_length(old_array) >= BPF_TRACE_MAX_PROGS) { 1958c8c088baSYonghong Song ret = -E2BIG; 1959c8c088baSYonghong Song goto unlock; 1960c8c088baSYonghong Song } 1961c8c088baSYonghong Song 1962e87c6bc3SYonghong Song ret = bpf_prog_array_copy(old_array, NULL, prog, &new_array); 1963e87c6bc3SYonghong Song if (ret < 0) 196407c41a29SYonghong Song goto unlock; 1965e87c6bc3SYonghong Song 1966e87c6bc3SYonghong Song /* set the new array to event->tp_event and set event->prog */ 1967e87c6bc3SYonghong Song event->prog = prog; 1968e87c6bc3SYonghong Song rcu_assign_pointer(event->tp_event->prog_array, new_array); 1969e87c6bc3SYonghong Song bpf_prog_array_free(old_array); 1970e87c6bc3SYonghong Song 197107c41a29SYonghong Song unlock: 1972e87c6bc3SYonghong Song mutex_unlock(&bpf_event_mutex); 1973e87c6bc3SYonghong Song return ret; 1974e87c6bc3SYonghong Song } 1975e87c6bc3SYonghong Song 1976e87c6bc3SYonghong Song void perf_event_detach_bpf_prog(struct perf_event *event) 1977e87c6bc3SYonghong Song { 1978e672db03SStanislav Fomichev struct bpf_prog_array *old_array; 1979e87c6bc3SYonghong Song struct bpf_prog_array *new_array; 1980e87c6bc3SYonghong Song int ret; 1981e87c6bc3SYonghong Song 1982e87c6bc3SYonghong Song mutex_lock(&bpf_event_mutex); 1983e87c6bc3SYonghong Song 1984e87c6bc3SYonghong Song if (!event->prog) 198507c41a29SYonghong Song goto unlock; 1986e87c6bc3SYonghong Song 1987e672db03SStanislav Fomichev old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); 1988e87c6bc3SYonghong Song ret = bpf_prog_array_copy(old_array, event->prog, NULL, &new_array); 1989170a7e3eSSean Young if (ret == -ENOENT) 1990170a7e3eSSean Young goto unlock; 1991e87c6bc3SYonghong Song if (ret < 0) { 1992e87c6bc3SYonghong Song bpf_prog_array_delete_safe(old_array, event->prog); 1993e87c6bc3SYonghong Song } else { 1994e87c6bc3SYonghong Song rcu_assign_pointer(event->tp_event->prog_array, new_array); 1995e87c6bc3SYonghong Song bpf_prog_array_free(old_array); 1996e87c6bc3SYonghong Song } 1997e87c6bc3SYonghong Song 1998e87c6bc3SYonghong Song bpf_prog_put(event->prog); 1999e87c6bc3SYonghong Song event->prog = NULL; 2000e87c6bc3SYonghong Song 200107c41a29SYonghong Song unlock: 2002e87c6bc3SYonghong Song mutex_unlock(&bpf_event_mutex); 2003e87c6bc3SYonghong Song } 2004f371b304SYonghong Song 2005f4e2298eSYonghong Song int perf_event_query_prog_array(struct perf_event *event, void __user *info) 2006f371b304SYonghong Song { 2007f371b304SYonghong Song struct perf_event_query_bpf __user *uquery = info; 2008f371b304SYonghong Song struct perf_event_query_bpf query = {}; 2009e672db03SStanislav Fomichev struct bpf_prog_array *progs; 20103a38bb98SYonghong Song u32 *ids, prog_cnt, ids_len; 2011f371b304SYonghong Song int ret; 2012f371b304SYonghong Song 2013031258daSAlexey Budankov if (!perfmon_capable()) 2014f371b304SYonghong Song return -EPERM; 2015f371b304SYonghong Song if (event->attr.type != PERF_TYPE_TRACEPOINT) 2016f371b304SYonghong Song return -EINVAL; 2017f371b304SYonghong Song if (copy_from_user(&query, uquery, sizeof(query))) 2018f371b304SYonghong Song return -EFAULT; 20193a38bb98SYonghong Song 20203a38bb98SYonghong Song ids_len = query.ids_len; 20213a38bb98SYonghong Song if (ids_len > BPF_TRACE_MAX_PROGS) 20229c481b90SDaniel Borkmann return -E2BIG; 20233a38bb98SYonghong Song ids = kcalloc(ids_len, sizeof(u32), GFP_USER | __GFP_NOWARN); 20243a38bb98SYonghong Song if (!ids) 20253a38bb98SYonghong Song return -ENOMEM; 20263a38bb98SYonghong Song /* 20273a38bb98SYonghong Song * The above kcalloc returns ZERO_SIZE_PTR when ids_len = 0, which 20283a38bb98SYonghong Song * is required when user only wants to check for uquery->prog_cnt. 20293a38bb98SYonghong Song * There is no need to check for it since the case is handled 20303a38bb98SYonghong Song * gracefully in bpf_prog_array_copy_info. 20313a38bb98SYonghong Song */ 2032f371b304SYonghong Song 2033f371b304SYonghong Song mutex_lock(&bpf_event_mutex); 2034e672db03SStanislav Fomichev progs = bpf_event_rcu_dereference(event->tp_event->prog_array); 2035e672db03SStanislav Fomichev ret = bpf_prog_array_copy_info(progs, ids, ids_len, &prog_cnt); 2036f371b304SYonghong Song mutex_unlock(&bpf_event_mutex); 2037f371b304SYonghong Song 20383a38bb98SYonghong Song if (copy_to_user(&uquery->prog_cnt, &prog_cnt, sizeof(prog_cnt)) || 20393a38bb98SYonghong Song copy_to_user(uquery->ids, ids, ids_len * sizeof(u32))) 20403a38bb98SYonghong Song ret = -EFAULT; 20413a38bb98SYonghong Song 20423a38bb98SYonghong Song kfree(ids); 2043f371b304SYonghong Song return ret; 2044f371b304SYonghong Song } 2045c4f6699dSAlexei Starovoitov 2046c4f6699dSAlexei Starovoitov extern struct bpf_raw_event_map __start__bpf_raw_tp[]; 2047c4f6699dSAlexei Starovoitov extern struct bpf_raw_event_map __stop__bpf_raw_tp[]; 2048c4f6699dSAlexei Starovoitov 2049a38d1107SMatt Mullins struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name) 2050c4f6699dSAlexei Starovoitov { 2051c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp = __start__bpf_raw_tp; 2052c4f6699dSAlexei Starovoitov 2053c4f6699dSAlexei Starovoitov for (; btp < __stop__bpf_raw_tp; btp++) { 2054c4f6699dSAlexei Starovoitov if (!strcmp(btp->tp->name, name)) 2055c4f6699dSAlexei Starovoitov return btp; 2056c4f6699dSAlexei Starovoitov } 2057a38d1107SMatt Mullins 2058a38d1107SMatt Mullins return bpf_get_raw_tracepoint_module(name); 2059a38d1107SMatt Mullins } 2060a38d1107SMatt Mullins 2061a38d1107SMatt Mullins void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp) 2062a38d1107SMatt Mullins { 2063*12cc126dSAndrii Nakryiko struct module *mod; 2064a38d1107SMatt Mullins 2065*12cc126dSAndrii Nakryiko preempt_disable(); 2066*12cc126dSAndrii Nakryiko mod = __module_address((unsigned long)btp); 2067a38d1107SMatt Mullins module_put(mod); 2068*12cc126dSAndrii Nakryiko preempt_enable(); 2069c4f6699dSAlexei Starovoitov } 2070c4f6699dSAlexei Starovoitov 2071c4f6699dSAlexei Starovoitov static __always_inline 2072c4f6699dSAlexei Starovoitov void __bpf_trace_run(struct bpf_prog *prog, u64 *args) 2073c4f6699dSAlexei Starovoitov { 2074f03efe49SThomas Gleixner cant_sleep(); 2075c4f6699dSAlexei Starovoitov rcu_read_lock(); 2076c4f6699dSAlexei Starovoitov (void) BPF_PROG_RUN(prog, args); 2077c4f6699dSAlexei Starovoitov rcu_read_unlock(); 2078c4f6699dSAlexei Starovoitov } 2079c4f6699dSAlexei Starovoitov 2080c4f6699dSAlexei Starovoitov #define UNPACK(...) __VA_ARGS__ 2081c4f6699dSAlexei Starovoitov #define REPEAT_1(FN, DL, X, ...) FN(X) 2082c4f6699dSAlexei Starovoitov #define REPEAT_2(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_1(FN, DL, __VA_ARGS__) 2083c4f6699dSAlexei Starovoitov #define REPEAT_3(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_2(FN, DL, __VA_ARGS__) 2084c4f6699dSAlexei Starovoitov #define REPEAT_4(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_3(FN, DL, __VA_ARGS__) 2085c4f6699dSAlexei Starovoitov #define REPEAT_5(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_4(FN, DL, __VA_ARGS__) 2086c4f6699dSAlexei Starovoitov #define REPEAT_6(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_5(FN, DL, __VA_ARGS__) 2087c4f6699dSAlexei Starovoitov #define REPEAT_7(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_6(FN, DL, __VA_ARGS__) 2088c4f6699dSAlexei Starovoitov #define REPEAT_8(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_7(FN, DL, __VA_ARGS__) 2089c4f6699dSAlexei Starovoitov #define REPEAT_9(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_8(FN, DL, __VA_ARGS__) 2090c4f6699dSAlexei Starovoitov #define REPEAT_10(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_9(FN, DL, __VA_ARGS__) 2091c4f6699dSAlexei Starovoitov #define REPEAT_11(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_10(FN, DL, __VA_ARGS__) 2092c4f6699dSAlexei Starovoitov #define REPEAT_12(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_11(FN, DL, __VA_ARGS__) 2093c4f6699dSAlexei Starovoitov #define REPEAT(X, FN, DL, ...) REPEAT_##X(FN, DL, __VA_ARGS__) 2094c4f6699dSAlexei Starovoitov 2095c4f6699dSAlexei Starovoitov #define SARG(X) u64 arg##X 2096c4f6699dSAlexei Starovoitov #define COPY(X) args[X] = arg##X 2097c4f6699dSAlexei Starovoitov 2098c4f6699dSAlexei Starovoitov #define __DL_COM (,) 2099c4f6699dSAlexei Starovoitov #define __DL_SEM (;) 2100c4f6699dSAlexei Starovoitov 2101c4f6699dSAlexei Starovoitov #define __SEQ_0_11 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 2102c4f6699dSAlexei Starovoitov 2103c4f6699dSAlexei Starovoitov #define BPF_TRACE_DEFN_x(x) \ 2104c4f6699dSAlexei Starovoitov void bpf_trace_run##x(struct bpf_prog *prog, \ 2105c4f6699dSAlexei Starovoitov REPEAT(x, SARG, __DL_COM, __SEQ_0_11)) \ 2106c4f6699dSAlexei Starovoitov { \ 2107c4f6699dSAlexei Starovoitov u64 args[x]; \ 2108c4f6699dSAlexei Starovoitov REPEAT(x, COPY, __DL_SEM, __SEQ_0_11); \ 2109c4f6699dSAlexei Starovoitov __bpf_trace_run(prog, args); \ 2110c4f6699dSAlexei Starovoitov } \ 2111c4f6699dSAlexei Starovoitov EXPORT_SYMBOL_GPL(bpf_trace_run##x) 2112c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(1); 2113c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(2); 2114c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(3); 2115c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(4); 2116c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(5); 2117c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(6); 2118c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(7); 2119c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(8); 2120c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(9); 2121c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(10); 2122c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(11); 2123c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(12); 2124c4f6699dSAlexei Starovoitov 2125c4f6699dSAlexei Starovoitov static int __bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog) 2126c4f6699dSAlexei Starovoitov { 2127c4f6699dSAlexei Starovoitov struct tracepoint *tp = btp->tp; 2128c4f6699dSAlexei Starovoitov 2129c4f6699dSAlexei Starovoitov /* 2130c4f6699dSAlexei Starovoitov * check that program doesn't access arguments beyond what's 2131c4f6699dSAlexei Starovoitov * available in this tracepoint 2132c4f6699dSAlexei Starovoitov */ 2133c4f6699dSAlexei Starovoitov if (prog->aux->max_ctx_offset > btp->num_args * sizeof(u64)) 2134c4f6699dSAlexei Starovoitov return -EINVAL; 2135c4f6699dSAlexei Starovoitov 21369df1c28bSMatt Mullins if (prog->aux->max_tp_access > btp->writable_size) 21379df1c28bSMatt Mullins return -EINVAL; 21389df1c28bSMatt Mullins 2139c4f6699dSAlexei Starovoitov return tracepoint_probe_register(tp, (void *)btp->bpf_func, prog); 2140c4f6699dSAlexei Starovoitov } 2141c4f6699dSAlexei Starovoitov 2142c4f6699dSAlexei Starovoitov int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog) 2143c4f6699dSAlexei Starovoitov { 2144e16ec340SAlexei Starovoitov return __bpf_probe_register(btp, prog); 2145c4f6699dSAlexei Starovoitov } 2146c4f6699dSAlexei Starovoitov 2147c4f6699dSAlexei Starovoitov int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog) 2148c4f6699dSAlexei Starovoitov { 2149e16ec340SAlexei Starovoitov return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, prog); 2150c4f6699dSAlexei Starovoitov } 215141bdc4b4SYonghong Song 215241bdc4b4SYonghong Song int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, 215341bdc4b4SYonghong Song u32 *fd_type, const char **buf, 215441bdc4b4SYonghong Song u64 *probe_offset, u64 *probe_addr) 215541bdc4b4SYonghong Song { 215641bdc4b4SYonghong Song bool is_tracepoint, is_syscall_tp; 215741bdc4b4SYonghong Song struct bpf_prog *prog; 215841bdc4b4SYonghong Song int flags, err = 0; 215941bdc4b4SYonghong Song 216041bdc4b4SYonghong Song prog = event->prog; 216141bdc4b4SYonghong Song if (!prog) 216241bdc4b4SYonghong Song return -ENOENT; 216341bdc4b4SYonghong Song 216441bdc4b4SYonghong Song /* not supporting BPF_PROG_TYPE_PERF_EVENT yet */ 216541bdc4b4SYonghong Song if (prog->type == BPF_PROG_TYPE_PERF_EVENT) 216641bdc4b4SYonghong Song return -EOPNOTSUPP; 216741bdc4b4SYonghong Song 216841bdc4b4SYonghong Song *prog_id = prog->aux->id; 216941bdc4b4SYonghong Song flags = event->tp_event->flags; 217041bdc4b4SYonghong Song is_tracepoint = flags & TRACE_EVENT_FL_TRACEPOINT; 217141bdc4b4SYonghong Song is_syscall_tp = is_syscall_trace_event(event->tp_event); 217241bdc4b4SYonghong Song 217341bdc4b4SYonghong Song if (is_tracepoint || is_syscall_tp) { 217441bdc4b4SYonghong Song *buf = is_tracepoint ? event->tp_event->tp->name 217541bdc4b4SYonghong Song : event->tp_event->name; 217641bdc4b4SYonghong Song *fd_type = BPF_FD_TYPE_TRACEPOINT; 217741bdc4b4SYonghong Song *probe_offset = 0x0; 217841bdc4b4SYonghong Song *probe_addr = 0x0; 217941bdc4b4SYonghong Song } else { 218041bdc4b4SYonghong Song /* kprobe/uprobe */ 218141bdc4b4SYonghong Song err = -EOPNOTSUPP; 218241bdc4b4SYonghong Song #ifdef CONFIG_KPROBE_EVENTS 218341bdc4b4SYonghong Song if (flags & TRACE_EVENT_FL_KPROBE) 218441bdc4b4SYonghong Song err = bpf_get_kprobe_info(event, fd_type, buf, 218541bdc4b4SYonghong Song probe_offset, probe_addr, 218641bdc4b4SYonghong Song event->attr.type == PERF_TYPE_TRACEPOINT); 218741bdc4b4SYonghong Song #endif 218841bdc4b4SYonghong Song #ifdef CONFIG_UPROBE_EVENTS 218941bdc4b4SYonghong Song if (flags & TRACE_EVENT_FL_UPROBE) 219041bdc4b4SYonghong Song err = bpf_get_uprobe_info(event, fd_type, buf, 219141bdc4b4SYonghong Song probe_offset, 219241bdc4b4SYonghong Song event->attr.type == PERF_TYPE_TRACEPOINT); 219341bdc4b4SYonghong Song #endif 219441bdc4b4SYonghong Song } 219541bdc4b4SYonghong Song 219641bdc4b4SYonghong Song return err; 219741bdc4b4SYonghong Song } 2198a38d1107SMatt Mullins 21999db1ff0aSYonghong Song static int __init send_signal_irq_work_init(void) 22009db1ff0aSYonghong Song { 22019db1ff0aSYonghong Song int cpu; 22029db1ff0aSYonghong Song struct send_signal_irq_work *work; 22039db1ff0aSYonghong Song 22049db1ff0aSYonghong Song for_each_possible_cpu(cpu) { 22059db1ff0aSYonghong Song work = per_cpu_ptr(&send_signal_work, cpu); 22069db1ff0aSYonghong Song init_irq_work(&work->irq_work, do_bpf_send_signal); 22079db1ff0aSYonghong Song } 22089db1ff0aSYonghong Song return 0; 22099db1ff0aSYonghong Song } 22109db1ff0aSYonghong Song 22119db1ff0aSYonghong Song subsys_initcall(send_signal_irq_work_init); 22129db1ff0aSYonghong Song 2213a38d1107SMatt Mullins #ifdef CONFIG_MODULES 2214390e99cfSStanislav Fomichev static int bpf_event_notify(struct notifier_block *nb, unsigned long op, 2215390e99cfSStanislav Fomichev void *module) 2216a38d1107SMatt Mullins { 2217a38d1107SMatt Mullins struct bpf_trace_module *btm, *tmp; 2218a38d1107SMatt Mullins struct module *mod = module; 22190340a6b7SPeter Zijlstra int ret = 0; 2220a38d1107SMatt Mullins 2221a38d1107SMatt Mullins if (mod->num_bpf_raw_events == 0 || 2222a38d1107SMatt Mullins (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING)) 22230340a6b7SPeter Zijlstra goto out; 2224a38d1107SMatt Mullins 2225a38d1107SMatt Mullins mutex_lock(&bpf_module_mutex); 2226a38d1107SMatt Mullins 2227a38d1107SMatt Mullins switch (op) { 2228a38d1107SMatt Mullins case MODULE_STATE_COMING: 2229a38d1107SMatt Mullins btm = kzalloc(sizeof(*btm), GFP_KERNEL); 2230a38d1107SMatt Mullins if (btm) { 2231a38d1107SMatt Mullins btm->module = module; 2232a38d1107SMatt Mullins list_add(&btm->list, &bpf_trace_modules); 22330340a6b7SPeter Zijlstra } else { 22340340a6b7SPeter Zijlstra ret = -ENOMEM; 2235a38d1107SMatt Mullins } 2236a38d1107SMatt Mullins break; 2237a38d1107SMatt Mullins case MODULE_STATE_GOING: 2238a38d1107SMatt Mullins list_for_each_entry_safe(btm, tmp, &bpf_trace_modules, list) { 2239a38d1107SMatt Mullins if (btm->module == module) { 2240a38d1107SMatt Mullins list_del(&btm->list); 2241a38d1107SMatt Mullins kfree(btm); 2242a38d1107SMatt Mullins break; 2243a38d1107SMatt Mullins } 2244a38d1107SMatt Mullins } 2245a38d1107SMatt Mullins break; 2246a38d1107SMatt Mullins } 2247a38d1107SMatt Mullins 2248a38d1107SMatt Mullins mutex_unlock(&bpf_module_mutex); 2249a38d1107SMatt Mullins 22500340a6b7SPeter Zijlstra out: 22510340a6b7SPeter Zijlstra return notifier_from_errno(ret); 2252a38d1107SMatt Mullins } 2253a38d1107SMatt Mullins 2254a38d1107SMatt Mullins static struct notifier_block bpf_module_nb = { 2255a38d1107SMatt Mullins .notifier_call = bpf_event_notify, 2256a38d1107SMatt Mullins }; 2257a38d1107SMatt Mullins 2258390e99cfSStanislav Fomichev static int __init bpf_event_init(void) 2259a38d1107SMatt Mullins { 2260a38d1107SMatt Mullins register_module_notifier(&bpf_module_nb); 2261a38d1107SMatt Mullins return 0; 2262a38d1107SMatt Mullins } 2263a38d1107SMatt Mullins 2264a38d1107SMatt Mullins fs_initcall(bpf_event_init); 2265a38d1107SMatt Mullins #endif /* CONFIG_MODULES */ 2266