1179a0cc4SSteven Rostedt (VMware) // SPDX-License-Identifier: GPL-2.0 22541517cSAlexei Starovoitov /* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com 30515e599SAlexei Starovoitov * Copyright (c) 2016 Facebook 42541517cSAlexei Starovoitov */ 52541517cSAlexei Starovoitov #include <linux/kernel.h> 62541517cSAlexei Starovoitov #include <linux/types.h> 72541517cSAlexei Starovoitov #include <linux/slab.h> 82541517cSAlexei Starovoitov #include <linux/bpf.h> 90515e599SAlexei Starovoitov #include <linux/bpf_perf_event.h> 10c4d0bfb4SAlan Maguire #include <linux/btf.h> 112541517cSAlexei Starovoitov #include <linux/filter.h> 122541517cSAlexei Starovoitov #include <linux/uaccess.h> 139c959c86SAlexei Starovoitov #include <linux/ctype.h> 149802d865SJosef Bacik #include <linux/kprobes.h> 15ac5a72eaSAlan Maguire #include <linux/spinlock.h> 1641bdc4b4SYonghong Song #include <linux/syscalls.h> 17540adea3SMasami Hiramatsu #include <linux/error-injection.h> 18c9a0f3b8SJiri Olsa #include <linux/btf_ids.h> 196f100640SKP Singh #include <linux/bpf_lsm.h> 206f100640SKP Singh 218e4597c6SMartin KaFai Lau #include <net/bpf_sk_storage.h> 229802d865SJosef Bacik 23c4d0bfb4SAlan Maguire #include <uapi/linux/bpf.h> 24c4d0bfb4SAlan Maguire #include <uapi/linux/btf.h> 25c4d0bfb4SAlan Maguire 26c7b6f29bSNadav Amit #include <asm/tlb.h> 27c7b6f29bSNadav Amit 289802d865SJosef Bacik #include "trace_probe.h" 292541517cSAlexei Starovoitov #include "trace.h" 302541517cSAlexei Starovoitov 31ac5a72eaSAlan Maguire #define CREATE_TRACE_POINTS 32ac5a72eaSAlan Maguire #include "bpf_trace.h" 33ac5a72eaSAlan Maguire 34e672db03SStanislav Fomichev #define bpf_event_rcu_dereference(p) \ 35e672db03SStanislav Fomichev rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex)) 36e672db03SStanislav Fomichev 37a38d1107SMatt Mullins #ifdef CONFIG_MODULES 38a38d1107SMatt Mullins struct bpf_trace_module { 39a38d1107SMatt Mullins struct module *module; 40a38d1107SMatt Mullins struct list_head list; 41a38d1107SMatt Mullins }; 42a38d1107SMatt Mullins 43a38d1107SMatt Mullins static LIST_HEAD(bpf_trace_modules); 44a38d1107SMatt Mullins static DEFINE_MUTEX(bpf_module_mutex); 45a38d1107SMatt Mullins 46a38d1107SMatt Mullins static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name) 47a38d1107SMatt Mullins { 48a38d1107SMatt Mullins struct bpf_raw_event_map *btp, *ret = NULL; 49a38d1107SMatt Mullins struct bpf_trace_module *btm; 50a38d1107SMatt Mullins unsigned int i; 51a38d1107SMatt Mullins 52a38d1107SMatt Mullins mutex_lock(&bpf_module_mutex); 53a38d1107SMatt Mullins list_for_each_entry(btm, &bpf_trace_modules, list) { 54a38d1107SMatt Mullins for (i = 0; i < btm->module->num_bpf_raw_events; ++i) { 55a38d1107SMatt Mullins btp = &btm->module->bpf_raw_events[i]; 56a38d1107SMatt Mullins if (!strcmp(btp->tp->name, name)) { 57a38d1107SMatt Mullins if (try_module_get(btm->module)) 58a38d1107SMatt Mullins ret = btp; 59a38d1107SMatt Mullins goto out; 60a38d1107SMatt Mullins } 61a38d1107SMatt Mullins } 62a38d1107SMatt Mullins } 63a38d1107SMatt Mullins out: 64a38d1107SMatt Mullins mutex_unlock(&bpf_module_mutex); 65a38d1107SMatt Mullins return ret; 66a38d1107SMatt Mullins } 67a38d1107SMatt Mullins #else 68a38d1107SMatt Mullins static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name) 69a38d1107SMatt Mullins { 70a38d1107SMatt Mullins return NULL; 71a38d1107SMatt Mullins } 72a38d1107SMatt Mullins #endif /* CONFIG_MODULES */ 73a38d1107SMatt Mullins 74035226b9SGianluca Borello u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 75c195651eSYonghong Song u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 76035226b9SGianluca Borello 77eb411377SAlan Maguire static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size, 78eb411377SAlan Maguire u64 flags, const struct btf **btf, 79eb411377SAlan Maguire s32 *btf_id); 80eb411377SAlan Maguire 812541517cSAlexei Starovoitov /** 822541517cSAlexei Starovoitov * trace_call_bpf - invoke BPF program 83e87c6bc3SYonghong Song * @call: tracepoint event 842541517cSAlexei Starovoitov * @ctx: opaque context pointer 852541517cSAlexei Starovoitov * 862541517cSAlexei Starovoitov * kprobe handlers execute BPF programs via this helper. 872541517cSAlexei Starovoitov * Can be used from static tracepoints in the future. 882541517cSAlexei Starovoitov * 892541517cSAlexei Starovoitov * Return: BPF programs always return an integer which is interpreted by 902541517cSAlexei Starovoitov * kprobe handler as: 912541517cSAlexei Starovoitov * 0 - return from kprobe (event is filtered out) 922541517cSAlexei Starovoitov * 1 - store kprobe event into ring buffer 932541517cSAlexei Starovoitov * Other values are reserved and currently alias to 1 942541517cSAlexei Starovoitov */ 95e87c6bc3SYonghong Song unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) 962541517cSAlexei Starovoitov { 972541517cSAlexei Starovoitov unsigned int ret; 982541517cSAlexei Starovoitov 99b0a81b94SThomas Gleixner cant_sleep(); 1002541517cSAlexei Starovoitov 1012541517cSAlexei Starovoitov if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { 1022541517cSAlexei Starovoitov /* 1032541517cSAlexei Starovoitov * since some bpf program is already running on this cpu, 1042541517cSAlexei Starovoitov * don't call into another bpf program (same or different) 1052541517cSAlexei Starovoitov * and don't send kprobe event into ring-buffer, 1062541517cSAlexei Starovoitov * so return zero here 1072541517cSAlexei Starovoitov */ 1082541517cSAlexei Starovoitov ret = 0; 1092541517cSAlexei Starovoitov goto out; 1102541517cSAlexei Starovoitov } 1112541517cSAlexei Starovoitov 112e87c6bc3SYonghong Song /* 113e87c6bc3SYonghong Song * Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock 114e87c6bc3SYonghong Song * to all call sites, we did a bpf_prog_array_valid() there to check 115e87c6bc3SYonghong Song * whether call->prog_array is empty or not, which is 1162b5894ccSQiujun Huang * a heuristic to speed up execution. 117e87c6bc3SYonghong Song * 118e87c6bc3SYonghong Song * If bpf_prog_array_valid() fetched prog_array was 119e87c6bc3SYonghong Song * non-NULL, we go into trace_call_bpf() and do the actual 120e87c6bc3SYonghong Song * proper rcu_dereference() under RCU lock. 121e87c6bc3SYonghong Song * If it turns out that prog_array is NULL then, we bail out. 122e87c6bc3SYonghong Song * For the opposite, if the bpf_prog_array_valid() fetched pointer 123e87c6bc3SYonghong Song * was NULL, you'll skip the prog_array with the risk of missing 124e87c6bc3SYonghong Song * out of events when it was updated in between this and the 125e87c6bc3SYonghong Song * rcu_dereference() which is accepted risk. 126e87c6bc3SYonghong Song */ 127e87c6bc3SYonghong Song ret = BPF_PROG_RUN_ARRAY_CHECK(call->prog_array, ctx, BPF_PROG_RUN); 1282541517cSAlexei Starovoitov 1292541517cSAlexei Starovoitov out: 1302541517cSAlexei Starovoitov __this_cpu_dec(bpf_prog_active); 1312541517cSAlexei Starovoitov 1322541517cSAlexei Starovoitov return ret; 1332541517cSAlexei Starovoitov } 1342541517cSAlexei Starovoitov 1359802d865SJosef Bacik #ifdef CONFIG_BPF_KPROBE_OVERRIDE 1369802d865SJosef Bacik BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc) 1379802d865SJosef Bacik { 1389802d865SJosef Bacik regs_set_return_value(regs, rc); 139540adea3SMasami Hiramatsu override_function_with_return(regs); 1409802d865SJosef Bacik return 0; 1419802d865SJosef Bacik } 1429802d865SJosef Bacik 1439802d865SJosef Bacik static const struct bpf_func_proto bpf_override_return_proto = { 1449802d865SJosef Bacik .func = bpf_override_return, 1459802d865SJosef Bacik .gpl_only = true, 1469802d865SJosef Bacik .ret_type = RET_INTEGER, 1479802d865SJosef Bacik .arg1_type = ARG_PTR_TO_CTX, 1489802d865SJosef Bacik .arg2_type = ARG_ANYTHING, 1499802d865SJosef Bacik }; 1509802d865SJosef Bacik #endif 1519802d865SJosef Bacik 1528d92db5cSChristoph Hellwig static __always_inline int 1538d92db5cSChristoph Hellwig bpf_probe_read_user_common(void *dst, u32 size, const void __user *unsafe_ptr) 1548d92db5cSChristoph Hellwig { 1558d92db5cSChristoph Hellwig int ret; 1568d92db5cSChristoph Hellwig 157c0ee37e8SChristoph Hellwig ret = copy_from_user_nofault(dst, unsafe_ptr, size); 1588d92db5cSChristoph Hellwig if (unlikely(ret < 0)) 1598d92db5cSChristoph Hellwig memset(dst, 0, size); 1608d92db5cSChristoph Hellwig return ret; 1618d92db5cSChristoph Hellwig } 1628d92db5cSChristoph Hellwig 1636ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_user, void *, dst, u32, size, 1646ae08ae3SDaniel Borkmann const void __user *, unsafe_ptr) 1652541517cSAlexei Starovoitov { 1668d92db5cSChristoph Hellwig return bpf_probe_read_user_common(dst, size, unsafe_ptr); 1672541517cSAlexei Starovoitov } 1682541517cSAlexei Starovoitov 169f470378cSJohn Fastabend const struct bpf_func_proto bpf_probe_read_user_proto = { 1706ae08ae3SDaniel Borkmann .func = bpf_probe_read_user, 1716ae08ae3SDaniel Borkmann .gpl_only = true, 1726ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 1736ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 1746ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 1756ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 1766ae08ae3SDaniel Borkmann }; 1776ae08ae3SDaniel Borkmann 1788d92db5cSChristoph Hellwig static __always_inline int 1798d92db5cSChristoph Hellwig bpf_probe_read_user_str_common(void *dst, u32 size, 1808d92db5cSChristoph Hellwig const void __user *unsafe_ptr) 1818d92db5cSChristoph Hellwig { 1828d92db5cSChristoph Hellwig int ret; 1838d92db5cSChristoph Hellwig 1846fa6d280SDaniel Xu /* 1856fa6d280SDaniel Xu * NB: We rely on strncpy_from_user() not copying junk past the NUL 1866fa6d280SDaniel Xu * terminator into `dst`. 1876fa6d280SDaniel Xu * 1886fa6d280SDaniel Xu * strncpy_from_user() does long-sized strides in the fast path. If the 1896fa6d280SDaniel Xu * strncpy does not mask out the bytes after the NUL in `unsafe_ptr`, 1906fa6d280SDaniel Xu * then there could be junk after the NUL in `dst`. If user takes `dst` 1916fa6d280SDaniel Xu * and keys a hash map with it, then semantically identical strings can 1926fa6d280SDaniel Xu * occupy multiple entries in the map. 1936fa6d280SDaniel Xu */ 1948d92db5cSChristoph Hellwig ret = strncpy_from_user_nofault(dst, unsafe_ptr, size); 1958d92db5cSChristoph Hellwig if (unlikely(ret < 0)) 1968d92db5cSChristoph Hellwig memset(dst, 0, size); 1978d92db5cSChristoph Hellwig return ret; 1988d92db5cSChristoph Hellwig } 1998d92db5cSChristoph Hellwig 2006ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_user_str, void *, dst, u32, size, 2016ae08ae3SDaniel Borkmann const void __user *, unsafe_ptr) 2026ae08ae3SDaniel Borkmann { 2038d92db5cSChristoph Hellwig return bpf_probe_read_user_str_common(dst, size, unsafe_ptr); 2046ae08ae3SDaniel Borkmann } 2056ae08ae3SDaniel Borkmann 206f470378cSJohn Fastabend const struct bpf_func_proto bpf_probe_read_user_str_proto = { 2076ae08ae3SDaniel Borkmann .func = bpf_probe_read_user_str, 2086ae08ae3SDaniel Borkmann .gpl_only = true, 2096ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 2106ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 2116ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 2126ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 2136ae08ae3SDaniel Borkmann }; 2146ae08ae3SDaniel Borkmann 2156ae08ae3SDaniel Borkmann static __always_inline int 2168d92db5cSChristoph Hellwig bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr) 2176ae08ae3SDaniel Borkmann { 218*ff40e510SDaniel Borkmann int ret; 2196ae08ae3SDaniel Borkmann 220fe557319SChristoph Hellwig ret = copy_from_kernel_nofault(dst, unsafe_ptr, size); 2216ae08ae3SDaniel Borkmann if (unlikely(ret < 0)) 2226ae08ae3SDaniel Borkmann memset(dst, 0, size); 2236ae08ae3SDaniel Borkmann return ret; 2246ae08ae3SDaniel Borkmann } 2256ae08ae3SDaniel Borkmann 2266ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_kernel, void *, dst, u32, size, 2276ae08ae3SDaniel Borkmann const void *, unsafe_ptr) 2286ae08ae3SDaniel Borkmann { 2298d92db5cSChristoph Hellwig return bpf_probe_read_kernel_common(dst, size, unsafe_ptr); 2306ae08ae3SDaniel Borkmann } 2316ae08ae3SDaniel Borkmann 232f470378cSJohn Fastabend const struct bpf_func_proto bpf_probe_read_kernel_proto = { 2336ae08ae3SDaniel Borkmann .func = bpf_probe_read_kernel, 2346ae08ae3SDaniel Borkmann .gpl_only = true, 2356ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 2366ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 2376ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 2386ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 2396ae08ae3SDaniel Borkmann }; 2406ae08ae3SDaniel Borkmann 2416ae08ae3SDaniel Borkmann static __always_inline int 2428d92db5cSChristoph Hellwig bpf_probe_read_kernel_str_common(void *dst, u32 size, const void *unsafe_ptr) 2436ae08ae3SDaniel Borkmann { 244*ff40e510SDaniel Borkmann int ret; 2458d92db5cSChristoph Hellwig 2466ae08ae3SDaniel Borkmann /* 2478d92db5cSChristoph Hellwig * The strncpy_from_kernel_nofault() call will likely not fill the 2488d92db5cSChristoph Hellwig * entire buffer, but that's okay in this circumstance as we're probing 2496ae08ae3SDaniel Borkmann * arbitrary memory anyway similar to bpf_probe_read_*() and might 2506ae08ae3SDaniel Borkmann * as well probe the stack. Thus, memory is explicitly cleared 2516ae08ae3SDaniel Borkmann * only in error case, so that improper users ignoring return 2526ae08ae3SDaniel Borkmann * code altogether don't copy garbage; otherwise length of string 2536ae08ae3SDaniel Borkmann * is returned that can be used for bpf_perf_event_output() et al. 2546ae08ae3SDaniel Borkmann */ 2558d92db5cSChristoph Hellwig ret = strncpy_from_kernel_nofault(dst, unsafe_ptr, size); 2566ae08ae3SDaniel Borkmann if (unlikely(ret < 0)) 2576ae08ae3SDaniel Borkmann memset(dst, 0, size); 2586ae08ae3SDaniel Borkmann return ret; 2596ae08ae3SDaniel Borkmann } 2606ae08ae3SDaniel Borkmann 2616ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_kernel_str, void *, dst, u32, size, 2626ae08ae3SDaniel Borkmann const void *, unsafe_ptr) 2636ae08ae3SDaniel Borkmann { 2648d92db5cSChristoph Hellwig return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr); 2656ae08ae3SDaniel Borkmann } 2666ae08ae3SDaniel Borkmann 267f470378cSJohn Fastabend const struct bpf_func_proto bpf_probe_read_kernel_str_proto = { 2686ae08ae3SDaniel Borkmann .func = bpf_probe_read_kernel_str, 2696ae08ae3SDaniel Borkmann .gpl_only = true, 2706ae08ae3SDaniel Borkmann .ret_type = RET_INTEGER, 2716ae08ae3SDaniel Borkmann .arg1_type = ARG_PTR_TO_UNINIT_MEM, 2726ae08ae3SDaniel Borkmann .arg2_type = ARG_CONST_SIZE_OR_ZERO, 2736ae08ae3SDaniel Borkmann .arg3_type = ARG_ANYTHING, 2746ae08ae3SDaniel Borkmann }; 2756ae08ae3SDaniel Borkmann 2768d92db5cSChristoph Hellwig #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE 2778d92db5cSChristoph Hellwig BPF_CALL_3(bpf_probe_read_compat, void *, dst, u32, size, 2788d92db5cSChristoph Hellwig const void *, unsafe_ptr) 2798d92db5cSChristoph Hellwig { 2808d92db5cSChristoph Hellwig if ((unsigned long)unsafe_ptr < TASK_SIZE) { 2818d92db5cSChristoph Hellwig return bpf_probe_read_user_common(dst, size, 2828d92db5cSChristoph Hellwig (__force void __user *)unsafe_ptr); 2838d92db5cSChristoph Hellwig } 2848d92db5cSChristoph Hellwig return bpf_probe_read_kernel_common(dst, size, unsafe_ptr); 2858d92db5cSChristoph Hellwig } 2868d92db5cSChristoph Hellwig 2878d92db5cSChristoph Hellwig static const struct bpf_func_proto bpf_probe_read_compat_proto = { 2888d92db5cSChristoph Hellwig .func = bpf_probe_read_compat, 2898d92db5cSChristoph Hellwig .gpl_only = true, 2908d92db5cSChristoph Hellwig .ret_type = RET_INTEGER, 2918d92db5cSChristoph Hellwig .arg1_type = ARG_PTR_TO_UNINIT_MEM, 2928d92db5cSChristoph Hellwig .arg2_type = ARG_CONST_SIZE_OR_ZERO, 2938d92db5cSChristoph Hellwig .arg3_type = ARG_ANYTHING, 2948d92db5cSChristoph Hellwig }; 2958d92db5cSChristoph Hellwig 2966ae08ae3SDaniel Borkmann BPF_CALL_3(bpf_probe_read_compat_str, void *, dst, u32, size, 2976ae08ae3SDaniel Borkmann const void *, unsafe_ptr) 2986ae08ae3SDaniel Borkmann { 2998d92db5cSChristoph Hellwig if ((unsigned long)unsafe_ptr < TASK_SIZE) { 3008d92db5cSChristoph Hellwig return bpf_probe_read_user_str_common(dst, size, 3018d92db5cSChristoph Hellwig (__force void __user *)unsafe_ptr); 3028d92db5cSChristoph Hellwig } 3038d92db5cSChristoph Hellwig return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr); 3046ae08ae3SDaniel Borkmann } 3056ae08ae3SDaniel Borkmann 3066ae08ae3SDaniel Borkmann static const struct bpf_func_proto bpf_probe_read_compat_str_proto = { 3076ae08ae3SDaniel Borkmann .func = bpf_probe_read_compat_str, 3082541517cSAlexei Starovoitov .gpl_only = true, 3092541517cSAlexei Starovoitov .ret_type = RET_INTEGER, 31039f19ebbSAlexei Starovoitov .arg1_type = ARG_PTR_TO_UNINIT_MEM, 3119c019e2bSYonghong Song .arg2_type = ARG_CONST_SIZE_OR_ZERO, 3122541517cSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 3132541517cSAlexei Starovoitov }; 3148d92db5cSChristoph Hellwig #endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */ 3152541517cSAlexei Starovoitov 316eb1b6688SDaniel Borkmann BPF_CALL_3(bpf_probe_write_user, void __user *, unsafe_ptr, const void *, src, 317f3694e00SDaniel Borkmann u32, size) 31896ae5227SSargun Dhillon { 31996ae5227SSargun Dhillon /* 32096ae5227SSargun Dhillon * Ensure we're in user context which is safe for the helper to 32196ae5227SSargun Dhillon * run. This helper has no business in a kthread. 32296ae5227SSargun Dhillon * 32396ae5227SSargun Dhillon * access_ok() should prevent writing to non-user memory, but in 32496ae5227SSargun Dhillon * some situations (nommu, temporary switch, etc) access_ok() does 32596ae5227SSargun Dhillon * not provide enough validation, hence the check on KERNEL_DS. 326c7b6f29bSNadav Amit * 327c7b6f29bSNadav Amit * nmi_uaccess_okay() ensures the probe is not run in an interim 328c7b6f29bSNadav Amit * state, when the task or mm are switched. This is specifically 329c7b6f29bSNadav Amit * required to prevent the use of temporary mm. 33096ae5227SSargun Dhillon */ 33196ae5227SSargun Dhillon 33296ae5227SSargun Dhillon if (unlikely(in_interrupt() || 33396ae5227SSargun Dhillon current->flags & (PF_KTHREAD | PF_EXITING))) 33496ae5227SSargun Dhillon return -EPERM; 335db68ce10SAl Viro if (unlikely(uaccess_kernel())) 33696ae5227SSargun Dhillon return -EPERM; 337c7b6f29bSNadav Amit if (unlikely(!nmi_uaccess_okay())) 338c7b6f29bSNadav Amit return -EPERM; 33996ae5227SSargun Dhillon 340c0ee37e8SChristoph Hellwig return copy_to_user_nofault(unsafe_ptr, src, size); 34196ae5227SSargun Dhillon } 34296ae5227SSargun Dhillon 34396ae5227SSargun Dhillon static const struct bpf_func_proto bpf_probe_write_user_proto = { 34496ae5227SSargun Dhillon .func = bpf_probe_write_user, 34596ae5227SSargun Dhillon .gpl_only = true, 34696ae5227SSargun Dhillon .ret_type = RET_INTEGER, 34796ae5227SSargun Dhillon .arg1_type = ARG_ANYTHING, 34839f19ebbSAlexei Starovoitov .arg2_type = ARG_PTR_TO_MEM, 34939f19ebbSAlexei Starovoitov .arg3_type = ARG_CONST_SIZE, 35096ae5227SSargun Dhillon }; 35196ae5227SSargun Dhillon 35296ae5227SSargun Dhillon static const struct bpf_func_proto *bpf_get_probe_write_proto(void) 35396ae5227SSargun Dhillon { 3542c78ee89SAlexei Starovoitov if (!capable(CAP_SYS_ADMIN)) 3552c78ee89SAlexei Starovoitov return NULL; 3562c78ee89SAlexei Starovoitov 35796ae5227SSargun Dhillon pr_warn_ratelimited("%s[%d] is installing a program with bpf_probe_write_user helper that may corrupt user memory!", 35896ae5227SSargun Dhillon current->comm, task_pid_nr(current)); 35996ae5227SSargun Dhillon 36096ae5227SSargun Dhillon return &bpf_probe_write_user_proto; 36196ae5227SSargun Dhillon } 36296ae5227SSargun Dhillon 363ac5a72eaSAlan Maguire static DEFINE_RAW_SPINLOCK(trace_printk_lock); 364ac5a72eaSAlan Maguire 365d9c9e4dbSFlorent Revest #define MAX_TRACE_PRINTK_VARARGS 3 366ac5a72eaSAlan Maguire #define BPF_TRACE_PRINTK_SIZE 1024 367ac5a72eaSAlan Maguire 368f3694e00SDaniel Borkmann BPF_CALL_5(bpf_trace_printk, char *, fmt, u32, fmt_size, u64, arg1, 369f3694e00SDaniel Borkmann u64, arg2, u64, arg3) 3709c959c86SAlexei Starovoitov { 371d9c9e4dbSFlorent Revest u64 args[MAX_TRACE_PRINTK_VARARGS] = { arg1, arg2, arg3 }; 37248cac3f4SFlorent Revest u32 *bin_args; 373d9c9e4dbSFlorent Revest static char buf[BPF_TRACE_PRINTK_SIZE]; 374d9c9e4dbSFlorent Revest unsigned long flags; 375d9c9e4dbSFlorent Revest int ret; 3769c959c86SAlexei Starovoitov 37748cac3f4SFlorent Revest ret = bpf_bprintf_prepare(fmt, fmt_size, args, &bin_args, 378d9c9e4dbSFlorent Revest MAX_TRACE_PRINTK_VARARGS); 379d9c9e4dbSFlorent Revest if (ret < 0) 380d9c9e4dbSFlorent Revest return ret; 3819c959c86SAlexei Starovoitov 38238d26d89SFlorent Revest raw_spin_lock_irqsave(&trace_printk_lock, flags); 38348cac3f4SFlorent Revest ret = bstr_printf(buf, sizeof(buf), fmt, bin_args); 3849c959c86SAlexei Starovoitov 385d9c9e4dbSFlorent Revest trace_bpf_trace_printk(buf); 386d9c9e4dbSFlorent Revest raw_spin_unlock_irqrestore(&trace_printk_lock, flags); 3879c959c86SAlexei Starovoitov 38848cac3f4SFlorent Revest bpf_bprintf_cleanup(); 3899c959c86SAlexei Starovoitov 390d9c9e4dbSFlorent Revest return ret; 3919c959c86SAlexei Starovoitov } 3929c959c86SAlexei Starovoitov 3939c959c86SAlexei Starovoitov static const struct bpf_func_proto bpf_trace_printk_proto = { 3949c959c86SAlexei Starovoitov .func = bpf_trace_printk, 3959c959c86SAlexei Starovoitov .gpl_only = true, 3969c959c86SAlexei Starovoitov .ret_type = RET_INTEGER, 39739f19ebbSAlexei Starovoitov .arg1_type = ARG_PTR_TO_MEM, 39839f19ebbSAlexei Starovoitov .arg2_type = ARG_CONST_SIZE, 3999c959c86SAlexei Starovoitov }; 4009c959c86SAlexei Starovoitov 4010756ea3eSAlexei Starovoitov const struct bpf_func_proto *bpf_get_trace_printk_proto(void) 4020756ea3eSAlexei Starovoitov { 4030756ea3eSAlexei Starovoitov /* 404ac5a72eaSAlan Maguire * This program might be calling bpf_trace_printk, 405ac5a72eaSAlan Maguire * so enable the associated bpf_trace/bpf_trace_printk event. 406ac5a72eaSAlan Maguire * Repeat this each time as it is possible a user has 407ac5a72eaSAlan Maguire * disabled bpf_trace_printk events. By loading a program 408ac5a72eaSAlan Maguire * calling bpf_trace_printk() however the user has expressed 409ac5a72eaSAlan Maguire * the intent to see such events. 4100756ea3eSAlexei Starovoitov */ 411ac5a72eaSAlan Maguire if (trace_set_clr_event("bpf_trace", "bpf_trace_printk", 1)) 412ac5a72eaSAlan Maguire pr_warn_ratelimited("could not enable bpf_trace_printk events"); 4130756ea3eSAlexei Starovoitov 4140756ea3eSAlexei Starovoitov return &bpf_trace_printk_proto; 4150756ea3eSAlexei Starovoitov } 4160756ea3eSAlexei Starovoitov 417492e639fSYonghong Song #define MAX_SEQ_PRINTF_VARARGS 12 418492e639fSYonghong Song 419492e639fSYonghong Song BPF_CALL_5(bpf_seq_printf, struct seq_file *, m, char *, fmt, u32, fmt_size, 420492e639fSYonghong Song const void *, data, u32, data_len) 421492e639fSYonghong Song { 422d9c9e4dbSFlorent Revest int err, num_args; 42348cac3f4SFlorent Revest u32 *bin_args; 424492e639fSYonghong Song 425d9c9e4dbSFlorent Revest if (data_len & 7 || data_len > MAX_SEQ_PRINTF_VARARGS * 8 || 426d9c9e4dbSFlorent Revest (data_len && !data)) 427d9c9e4dbSFlorent Revest return -EINVAL; 428492e639fSYonghong Song num_args = data_len / 8; 429492e639fSYonghong Song 43048cac3f4SFlorent Revest err = bpf_bprintf_prepare(fmt, fmt_size, data, &bin_args, num_args); 431492e639fSYonghong Song if (err < 0) 432d9c9e4dbSFlorent Revest return err; 433492e639fSYonghong Song 43448cac3f4SFlorent Revest seq_bprintf(m, fmt, bin_args); 435492e639fSYonghong Song 43648cac3f4SFlorent Revest bpf_bprintf_cleanup(); 437d9c9e4dbSFlorent Revest 438d9c9e4dbSFlorent Revest return seq_has_overflowed(m) ? -EOVERFLOW : 0; 439492e639fSYonghong Song } 440492e639fSYonghong Song 4419436ef6eSLorenz Bauer BTF_ID_LIST_SINGLE(btf_seq_file_ids, struct, seq_file) 442c9a0f3b8SJiri Olsa 443492e639fSYonghong Song static const struct bpf_func_proto bpf_seq_printf_proto = { 444492e639fSYonghong Song .func = bpf_seq_printf, 445492e639fSYonghong Song .gpl_only = true, 446492e639fSYonghong Song .ret_type = RET_INTEGER, 447492e639fSYonghong Song .arg1_type = ARG_PTR_TO_BTF_ID, 4489436ef6eSLorenz Bauer .arg1_btf_id = &btf_seq_file_ids[0], 449492e639fSYonghong Song .arg2_type = ARG_PTR_TO_MEM, 450492e639fSYonghong Song .arg3_type = ARG_CONST_SIZE, 451492e639fSYonghong Song .arg4_type = ARG_PTR_TO_MEM_OR_NULL, 452492e639fSYonghong Song .arg5_type = ARG_CONST_SIZE_OR_ZERO, 453492e639fSYonghong Song }; 454492e639fSYonghong Song 455492e639fSYonghong Song BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len) 456492e639fSYonghong Song { 457492e639fSYonghong Song return seq_write(m, data, len) ? -EOVERFLOW : 0; 458492e639fSYonghong Song } 459492e639fSYonghong Song 460492e639fSYonghong Song static const struct bpf_func_proto bpf_seq_write_proto = { 461492e639fSYonghong Song .func = bpf_seq_write, 462492e639fSYonghong Song .gpl_only = true, 463492e639fSYonghong Song .ret_type = RET_INTEGER, 464492e639fSYonghong Song .arg1_type = ARG_PTR_TO_BTF_ID, 4659436ef6eSLorenz Bauer .arg1_btf_id = &btf_seq_file_ids[0], 466492e639fSYonghong Song .arg2_type = ARG_PTR_TO_MEM, 467492e639fSYonghong Song .arg3_type = ARG_CONST_SIZE_OR_ZERO, 468492e639fSYonghong Song }; 469492e639fSYonghong Song 470eb411377SAlan Maguire BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr, 471eb411377SAlan Maguire u32, btf_ptr_size, u64, flags) 472eb411377SAlan Maguire { 473eb411377SAlan Maguire const struct btf *btf; 474eb411377SAlan Maguire s32 btf_id; 475eb411377SAlan Maguire int ret; 476eb411377SAlan Maguire 477eb411377SAlan Maguire ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id); 478eb411377SAlan Maguire if (ret) 479eb411377SAlan Maguire return ret; 480eb411377SAlan Maguire 481eb411377SAlan Maguire return btf_type_seq_show_flags(btf, btf_id, ptr->ptr, m, flags); 482eb411377SAlan Maguire } 483eb411377SAlan Maguire 484eb411377SAlan Maguire static const struct bpf_func_proto bpf_seq_printf_btf_proto = { 485eb411377SAlan Maguire .func = bpf_seq_printf_btf, 486eb411377SAlan Maguire .gpl_only = true, 487eb411377SAlan Maguire .ret_type = RET_INTEGER, 488eb411377SAlan Maguire .arg1_type = ARG_PTR_TO_BTF_ID, 489eb411377SAlan Maguire .arg1_btf_id = &btf_seq_file_ids[0], 490eb411377SAlan Maguire .arg2_type = ARG_PTR_TO_MEM, 491eb411377SAlan Maguire .arg3_type = ARG_CONST_SIZE_OR_ZERO, 492eb411377SAlan Maguire .arg4_type = ARG_ANYTHING, 493d9847d31SAlexei Starovoitov }; 494d9847d31SAlexei Starovoitov 495908432caSYonghong Song static __always_inline int 496908432caSYonghong Song get_map_perf_counter(struct bpf_map *map, u64 flags, 497908432caSYonghong Song u64 *value, u64 *enabled, u64 *running) 49835578d79SKaixu Xia { 49935578d79SKaixu Xia struct bpf_array *array = container_of(map, struct bpf_array, map); 5006816a7ffSDaniel Borkmann unsigned int cpu = smp_processor_id(); 5016816a7ffSDaniel Borkmann u64 index = flags & BPF_F_INDEX_MASK; 5023b1efb19SDaniel Borkmann struct bpf_event_entry *ee; 50335578d79SKaixu Xia 5046816a7ffSDaniel Borkmann if (unlikely(flags & ~(BPF_F_INDEX_MASK))) 5056816a7ffSDaniel Borkmann return -EINVAL; 5066816a7ffSDaniel Borkmann if (index == BPF_F_CURRENT_CPU) 5076816a7ffSDaniel Borkmann index = cpu; 50835578d79SKaixu Xia if (unlikely(index >= array->map.max_entries)) 50935578d79SKaixu Xia return -E2BIG; 51035578d79SKaixu Xia 5113b1efb19SDaniel Borkmann ee = READ_ONCE(array->ptrs[index]); 5121ca1cc98SDaniel Borkmann if (!ee) 51335578d79SKaixu Xia return -ENOENT; 51435578d79SKaixu Xia 515908432caSYonghong Song return perf_event_read_local(ee->event, value, enabled, running); 516908432caSYonghong Song } 517908432caSYonghong Song 518908432caSYonghong Song BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags) 519908432caSYonghong Song { 520908432caSYonghong Song u64 value = 0; 521908432caSYonghong Song int err; 522908432caSYonghong Song 523908432caSYonghong Song err = get_map_perf_counter(map, flags, &value, NULL, NULL); 52435578d79SKaixu Xia /* 525f91840a3SAlexei Starovoitov * this api is ugly since we miss [-22..-2] range of valid 526f91840a3SAlexei Starovoitov * counter values, but that's uapi 52735578d79SKaixu Xia */ 528f91840a3SAlexei Starovoitov if (err) 529f91840a3SAlexei Starovoitov return err; 530f91840a3SAlexei Starovoitov return value; 53135578d79SKaixu Xia } 53235578d79SKaixu Xia 53362544ce8SAlexei Starovoitov static const struct bpf_func_proto bpf_perf_event_read_proto = { 53435578d79SKaixu Xia .func = bpf_perf_event_read, 5351075ef59SAlexei Starovoitov .gpl_only = true, 53635578d79SKaixu Xia .ret_type = RET_INTEGER, 53735578d79SKaixu Xia .arg1_type = ARG_CONST_MAP_PTR, 53835578d79SKaixu Xia .arg2_type = ARG_ANYTHING, 53935578d79SKaixu Xia }; 54035578d79SKaixu Xia 541908432caSYonghong Song BPF_CALL_4(bpf_perf_event_read_value, struct bpf_map *, map, u64, flags, 542908432caSYonghong Song struct bpf_perf_event_value *, buf, u32, size) 543908432caSYonghong Song { 544908432caSYonghong Song int err = -EINVAL; 545908432caSYonghong Song 546908432caSYonghong Song if (unlikely(size != sizeof(struct bpf_perf_event_value))) 547908432caSYonghong Song goto clear; 548908432caSYonghong Song err = get_map_perf_counter(map, flags, &buf->counter, &buf->enabled, 549908432caSYonghong Song &buf->running); 550908432caSYonghong Song if (unlikely(err)) 551908432caSYonghong Song goto clear; 552908432caSYonghong Song return 0; 553908432caSYonghong Song clear: 554908432caSYonghong Song memset(buf, 0, size); 555908432caSYonghong Song return err; 556908432caSYonghong Song } 557908432caSYonghong Song 558908432caSYonghong Song static const struct bpf_func_proto bpf_perf_event_read_value_proto = { 559908432caSYonghong Song .func = bpf_perf_event_read_value, 560908432caSYonghong Song .gpl_only = true, 561908432caSYonghong Song .ret_type = RET_INTEGER, 562908432caSYonghong Song .arg1_type = ARG_CONST_MAP_PTR, 563908432caSYonghong Song .arg2_type = ARG_ANYTHING, 564908432caSYonghong Song .arg3_type = ARG_PTR_TO_UNINIT_MEM, 565908432caSYonghong Song .arg4_type = ARG_CONST_SIZE, 566908432caSYonghong Song }; 567908432caSYonghong Song 5688e7a3920SDaniel Borkmann static __always_inline u64 5698e7a3920SDaniel Borkmann __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, 570283ca526SDaniel Borkmann u64 flags, struct perf_sample_data *sd) 571a43eec30SAlexei Starovoitov { 572a43eec30SAlexei Starovoitov struct bpf_array *array = container_of(map, struct bpf_array, map); 573d7931330SDaniel Borkmann unsigned int cpu = smp_processor_id(); 5741e33759cSDaniel Borkmann u64 index = flags & BPF_F_INDEX_MASK; 5753b1efb19SDaniel Borkmann struct bpf_event_entry *ee; 576a43eec30SAlexei Starovoitov struct perf_event *event; 577a43eec30SAlexei Starovoitov 5781e33759cSDaniel Borkmann if (index == BPF_F_CURRENT_CPU) 579d7931330SDaniel Borkmann index = cpu; 580a43eec30SAlexei Starovoitov if (unlikely(index >= array->map.max_entries)) 581a43eec30SAlexei Starovoitov return -E2BIG; 582a43eec30SAlexei Starovoitov 5833b1efb19SDaniel Borkmann ee = READ_ONCE(array->ptrs[index]); 5841ca1cc98SDaniel Borkmann if (!ee) 585a43eec30SAlexei Starovoitov return -ENOENT; 586a43eec30SAlexei Starovoitov 5873b1efb19SDaniel Borkmann event = ee->event; 588a43eec30SAlexei Starovoitov if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE || 589a43eec30SAlexei Starovoitov event->attr.config != PERF_COUNT_SW_BPF_OUTPUT)) 590a43eec30SAlexei Starovoitov return -EINVAL; 591a43eec30SAlexei Starovoitov 592d7931330SDaniel Borkmann if (unlikely(event->oncpu != cpu)) 593a43eec30SAlexei Starovoitov return -EOPNOTSUPP; 594a43eec30SAlexei Starovoitov 59556201969SArnaldo Carvalho de Melo return perf_event_output(event, sd, regs); 596a43eec30SAlexei Starovoitov } 597a43eec30SAlexei Starovoitov 5989594dc3cSMatt Mullins /* 5999594dc3cSMatt Mullins * Support executing tracepoints in normal, irq, and nmi context that each call 6009594dc3cSMatt Mullins * bpf_perf_event_output 6019594dc3cSMatt Mullins */ 6029594dc3cSMatt Mullins struct bpf_trace_sample_data { 6039594dc3cSMatt Mullins struct perf_sample_data sds[3]; 6049594dc3cSMatt Mullins }; 6059594dc3cSMatt Mullins 6069594dc3cSMatt Mullins static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds); 6079594dc3cSMatt Mullins static DEFINE_PER_CPU(int, bpf_trace_nest_level); 608f3694e00SDaniel Borkmann BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map, 609f3694e00SDaniel Borkmann u64, flags, void *, data, u64, size) 6108e7a3920SDaniel Borkmann { 6119594dc3cSMatt Mullins struct bpf_trace_sample_data *sds = this_cpu_ptr(&bpf_trace_sds); 6129594dc3cSMatt Mullins int nest_level = this_cpu_inc_return(bpf_trace_nest_level); 6138e7a3920SDaniel Borkmann struct perf_raw_record raw = { 6148e7a3920SDaniel Borkmann .frag = { 6158e7a3920SDaniel Borkmann .size = size, 6168e7a3920SDaniel Borkmann .data = data, 6178e7a3920SDaniel Borkmann }, 6188e7a3920SDaniel Borkmann }; 6199594dc3cSMatt Mullins struct perf_sample_data *sd; 6209594dc3cSMatt Mullins int err; 6218e7a3920SDaniel Borkmann 6229594dc3cSMatt Mullins if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds))) { 6239594dc3cSMatt Mullins err = -EBUSY; 6249594dc3cSMatt Mullins goto out; 6259594dc3cSMatt Mullins } 6269594dc3cSMatt Mullins 6279594dc3cSMatt Mullins sd = &sds->sds[nest_level - 1]; 6289594dc3cSMatt Mullins 6299594dc3cSMatt Mullins if (unlikely(flags & ~(BPF_F_INDEX_MASK))) { 6309594dc3cSMatt Mullins err = -EINVAL; 6319594dc3cSMatt Mullins goto out; 6329594dc3cSMatt Mullins } 6338e7a3920SDaniel Borkmann 634283ca526SDaniel Borkmann perf_sample_data_init(sd, 0, 0); 635283ca526SDaniel Borkmann sd->raw = &raw; 636283ca526SDaniel Borkmann 6379594dc3cSMatt Mullins err = __bpf_perf_event_output(regs, map, flags, sd); 6389594dc3cSMatt Mullins 6399594dc3cSMatt Mullins out: 6409594dc3cSMatt Mullins this_cpu_dec(bpf_trace_nest_level); 6419594dc3cSMatt Mullins return err; 6428e7a3920SDaniel Borkmann } 6438e7a3920SDaniel Borkmann 644a43eec30SAlexei Starovoitov static const struct bpf_func_proto bpf_perf_event_output_proto = { 645a43eec30SAlexei Starovoitov .func = bpf_perf_event_output, 6461075ef59SAlexei Starovoitov .gpl_only = true, 647a43eec30SAlexei Starovoitov .ret_type = RET_INTEGER, 648a43eec30SAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 649a43eec30SAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 650a43eec30SAlexei Starovoitov .arg3_type = ARG_ANYTHING, 65139f19ebbSAlexei Starovoitov .arg4_type = ARG_PTR_TO_MEM, 652a60dd35dSGianluca Borello .arg5_type = ARG_CONST_SIZE_OR_ZERO, 653a43eec30SAlexei Starovoitov }; 654a43eec30SAlexei Starovoitov 655768fb61fSAllan Zhang static DEFINE_PER_CPU(int, bpf_event_output_nest_level); 656768fb61fSAllan Zhang struct bpf_nested_pt_regs { 657768fb61fSAllan Zhang struct pt_regs regs[3]; 658768fb61fSAllan Zhang }; 659768fb61fSAllan Zhang static DEFINE_PER_CPU(struct bpf_nested_pt_regs, bpf_pt_regs); 660768fb61fSAllan Zhang static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_misc_sds); 661bd570ff9SDaniel Borkmann 662555c8a86SDaniel Borkmann u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 663555c8a86SDaniel Borkmann void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy) 664bd570ff9SDaniel Borkmann { 665768fb61fSAllan Zhang int nest_level = this_cpu_inc_return(bpf_event_output_nest_level); 666555c8a86SDaniel Borkmann struct perf_raw_frag frag = { 667555c8a86SDaniel Borkmann .copy = ctx_copy, 668555c8a86SDaniel Borkmann .size = ctx_size, 669555c8a86SDaniel Borkmann .data = ctx, 670555c8a86SDaniel Borkmann }; 671555c8a86SDaniel Borkmann struct perf_raw_record raw = { 672555c8a86SDaniel Borkmann .frag = { 673183fc153SAndrew Morton { 674555c8a86SDaniel Borkmann .next = ctx_size ? &frag : NULL, 675183fc153SAndrew Morton }, 676555c8a86SDaniel Borkmann .size = meta_size, 677555c8a86SDaniel Borkmann .data = meta, 678555c8a86SDaniel Borkmann }, 679555c8a86SDaniel Borkmann }; 680768fb61fSAllan Zhang struct perf_sample_data *sd; 681768fb61fSAllan Zhang struct pt_regs *regs; 682768fb61fSAllan Zhang u64 ret; 683768fb61fSAllan Zhang 684768fb61fSAllan Zhang if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bpf_misc_sds.sds))) { 685768fb61fSAllan Zhang ret = -EBUSY; 686768fb61fSAllan Zhang goto out; 687768fb61fSAllan Zhang } 688768fb61fSAllan Zhang sd = this_cpu_ptr(&bpf_misc_sds.sds[nest_level - 1]); 689768fb61fSAllan Zhang regs = this_cpu_ptr(&bpf_pt_regs.regs[nest_level - 1]); 690bd570ff9SDaniel Borkmann 691bd570ff9SDaniel Borkmann perf_fetch_caller_regs(regs); 692283ca526SDaniel Borkmann perf_sample_data_init(sd, 0, 0); 693283ca526SDaniel Borkmann sd->raw = &raw; 694bd570ff9SDaniel Borkmann 695768fb61fSAllan Zhang ret = __bpf_perf_event_output(regs, map, flags, sd); 696768fb61fSAllan Zhang out: 697768fb61fSAllan Zhang this_cpu_dec(bpf_event_output_nest_level); 698768fb61fSAllan Zhang return ret; 699bd570ff9SDaniel Borkmann } 700bd570ff9SDaniel Borkmann 701f3694e00SDaniel Borkmann BPF_CALL_0(bpf_get_current_task) 702606274c5SAlexei Starovoitov { 703606274c5SAlexei Starovoitov return (long) current; 704606274c5SAlexei Starovoitov } 705606274c5SAlexei Starovoitov 706f470378cSJohn Fastabend const struct bpf_func_proto bpf_get_current_task_proto = { 707606274c5SAlexei Starovoitov .func = bpf_get_current_task, 708606274c5SAlexei Starovoitov .gpl_only = true, 709606274c5SAlexei Starovoitov .ret_type = RET_INTEGER, 710606274c5SAlexei Starovoitov }; 711606274c5SAlexei Starovoitov 7123ca1032aSKP Singh BPF_CALL_0(bpf_get_current_task_btf) 7133ca1032aSKP Singh { 7143ca1032aSKP Singh return (unsigned long) current; 7153ca1032aSKP Singh } 7163ca1032aSKP Singh 7173ca1032aSKP Singh BTF_ID_LIST_SINGLE(bpf_get_current_btf_ids, struct, task_struct) 7183ca1032aSKP Singh 7193ca1032aSKP Singh static const struct bpf_func_proto bpf_get_current_task_btf_proto = { 7203ca1032aSKP Singh .func = bpf_get_current_task_btf, 7213ca1032aSKP Singh .gpl_only = true, 7223ca1032aSKP Singh .ret_type = RET_PTR_TO_BTF_ID, 7233ca1032aSKP Singh .ret_btf_id = &bpf_get_current_btf_ids[0], 7243ca1032aSKP Singh }; 7253ca1032aSKP Singh 726f3694e00SDaniel Borkmann BPF_CALL_2(bpf_current_task_under_cgroup, struct bpf_map *, map, u32, idx) 72760d20f91SSargun Dhillon { 72860d20f91SSargun Dhillon struct bpf_array *array = container_of(map, struct bpf_array, map); 72960d20f91SSargun Dhillon struct cgroup *cgrp; 73060d20f91SSargun Dhillon 73160d20f91SSargun Dhillon if (unlikely(idx >= array->map.max_entries)) 73260d20f91SSargun Dhillon return -E2BIG; 73360d20f91SSargun Dhillon 73460d20f91SSargun Dhillon cgrp = READ_ONCE(array->ptrs[idx]); 73560d20f91SSargun Dhillon if (unlikely(!cgrp)) 73660d20f91SSargun Dhillon return -EAGAIN; 73760d20f91SSargun Dhillon 73860d20f91SSargun Dhillon return task_under_cgroup_hierarchy(current, cgrp); 73960d20f91SSargun Dhillon } 74060d20f91SSargun Dhillon 74160d20f91SSargun Dhillon static const struct bpf_func_proto bpf_current_task_under_cgroup_proto = { 74260d20f91SSargun Dhillon .func = bpf_current_task_under_cgroup, 74360d20f91SSargun Dhillon .gpl_only = false, 74460d20f91SSargun Dhillon .ret_type = RET_INTEGER, 74560d20f91SSargun Dhillon .arg1_type = ARG_CONST_MAP_PTR, 74660d20f91SSargun Dhillon .arg2_type = ARG_ANYTHING, 74760d20f91SSargun Dhillon }; 74860d20f91SSargun Dhillon 7498b401f9eSYonghong Song struct send_signal_irq_work { 7508b401f9eSYonghong Song struct irq_work irq_work; 7518b401f9eSYonghong Song struct task_struct *task; 7528b401f9eSYonghong Song u32 sig; 7538482941fSYonghong Song enum pid_type type; 7548b401f9eSYonghong Song }; 7558b401f9eSYonghong Song 7568b401f9eSYonghong Song static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work); 7578b401f9eSYonghong Song 7588b401f9eSYonghong Song static void do_bpf_send_signal(struct irq_work *entry) 7598b401f9eSYonghong Song { 7608b401f9eSYonghong Song struct send_signal_irq_work *work; 7618b401f9eSYonghong Song 7628b401f9eSYonghong Song work = container_of(entry, struct send_signal_irq_work, irq_work); 7638482941fSYonghong Song group_send_sig_info(work->sig, SEND_SIG_PRIV, work->task, work->type); 7648b401f9eSYonghong Song } 7658b401f9eSYonghong Song 7668482941fSYonghong Song static int bpf_send_signal_common(u32 sig, enum pid_type type) 7678b401f9eSYonghong Song { 7688b401f9eSYonghong Song struct send_signal_irq_work *work = NULL; 7698b401f9eSYonghong Song 7708b401f9eSYonghong Song /* Similar to bpf_probe_write_user, task needs to be 7718b401f9eSYonghong Song * in a sound condition and kernel memory access be 7728b401f9eSYonghong Song * permitted in order to send signal to the current 7738b401f9eSYonghong Song * task. 7748b401f9eSYonghong Song */ 7758b401f9eSYonghong Song if (unlikely(current->flags & (PF_KTHREAD | PF_EXITING))) 7768b401f9eSYonghong Song return -EPERM; 7778b401f9eSYonghong Song if (unlikely(uaccess_kernel())) 7788b401f9eSYonghong Song return -EPERM; 7798b401f9eSYonghong Song if (unlikely(!nmi_uaccess_okay())) 7808b401f9eSYonghong Song return -EPERM; 7818b401f9eSYonghong Song 7821bc7896eSYonghong Song if (irqs_disabled()) { 783e1afb702SYonghong Song /* Do an early check on signal validity. Otherwise, 784e1afb702SYonghong Song * the error is lost in deferred irq_work. 785e1afb702SYonghong Song */ 786e1afb702SYonghong Song if (unlikely(!valid_signal(sig))) 787e1afb702SYonghong Song return -EINVAL; 788e1afb702SYonghong Song 7898b401f9eSYonghong Song work = this_cpu_ptr(&send_signal_work); 7907a9f50a0SPeter Zijlstra if (irq_work_is_busy(&work->irq_work)) 7918b401f9eSYonghong Song return -EBUSY; 7928b401f9eSYonghong Song 7938b401f9eSYonghong Song /* Add the current task, which is the target of sending signal, 7948b401f9eSYonghong Song * to the irq_work. The current task may change when queued 7958b401f9eSYonghong Song * irq works get executed. 7968b401f9eSYonghong Song */ 7978b401f9eSYonghong Song work->task = current; 7988b401f9eSYonghong Song work->sig = sig; 7998482941fSYonghong Song work->type = type; 8008b401f9eSYonghong Song irq_work_queue(&work->irq_work); 8018b401f9eSYonghong Song return 0; 8028b401f9eSYonghong Song } 8038b401f9eSYonghong Song 8048482941fSYonghong Song return group_send_sig_info(sig, SEND_SIG_PRIV, current, type); 8058482941fSYonghong Song } 8068482941fSYonghong Song 8078482941fSYonghong Song BPF_CALL_1(bpf_send_signal, u32, sig) 8088482941fSYonghong Song { 8098482941fSYonghong Song return bpf_send_signal_common(sig, PIDTYPE_TGID); 8108b401f9eSYonghong Song } 8118b401f9eSYonghong Song 8128b401f9eSYonghong Song static const struct bpf_func_proto bpf_send_signal_proto = { 8138b401f9eSYonghong Song .func = bpf_send_signal, 8148b401f9eSYonghong Song .gpl_only = false, 8158b401f9eSYonghong Song .ret_type = RET_INTEGER, 8168b401f9eSYonghong Song .arg1_type = ARG_ANYTHING, 8178b401f9eSYonghong Song }; 8188b401f9eSYonghong Song 8198482941fSYonghong Song BPF_CALL_1(bpf_send_signal_thread, u32, sig) 8208482941fSYonghong Song { 8218482941fSYonghong Song return bpf_send_signal_common(sig, PIDTYPE_PID); 8228482941fSYonghong Song } 8238482941fSYonghong Song 8248482941fSYonghong Song static const struct bpf_func_proto bpf_send_signal_thread_proto = { 8258482941fSYonghong Song .func = bpf_send_signal_thread, 8268482941fSYonghong Song .gpl_only = false, 8278482941fSYonghong Song .ret_type = RET_INTEGER, 8288482941fSYonghong Song .arg1_type = ARG_ANYTHING, 8298482941fSYonghong Song }; 8308482941fSYonghong Song 8316e22ab9dSJiri Olsa BPF_CALL_3(bpf_d_path, struct path *, path, char *, buf, u32, sz) 8326e22ab9dSJiri Olsa { 8336e22ab9dSJiri Olsa long len; 8346e22ab9dSJiri Olsa char *p; 8356e22ab9dSJiri Olsa 8366e22ab9dSJiri Olsa if (!sz) 8376e22ab9dSJiri Olsa return 0; 8386e22ab9dSJiri Olsa 8396e22ab9dSJiri Olsa p = d_path(path, buf, sz); 8406e22ab9dSJiri Olsa if (IS_ERR(p)) { 8416e22ab9dSJiri Olsa len = PTR_ERR(p); 8426e22ab9dSJiri Olsa } else { 8436e22ab9dSJiri Olsa len = buf + sz - p; 8446e22ab9dSJiri Olsa memmove(buf, p, len); 8456e22ab9dSJiri Olsa } 8466e22ab9dSJiri Olsa 8476e22ab9dSJiri Olsa return len; 8486e22ab9dSJiri Olsa } 8496e22ab9dSJiri Olsa 8506e22ab9dSJiri Olsa BTF_SET_START(btf_allowlist_d_path) 851a8a71796SJiri Olsa #ifdef CONFIG_SECURITY 852a8a71796SJiri Olsa BTF_ID(func, security_file_permission) 853a8a71796SJiri Olsa BTF_ID(func, security_inode_getattr) 854a8a71796SJiri Olsa BTF_ID(func, security_file_open) 855a8a71796SJiri Olsa #endif 856a8a71796SJiri Olsa #ifdef CONFIG_SECURITY_PATH 857a8a71796SJiri Olsa BTF_ID(func, security_path_truncate) 858a8a71796SJiri Olsa #endif 8596e22ab9dSJiri Olsa BTF_ID(func, vfs_truncate) 8606e22ab9dSJiri Olsa BTF_ID(func, vfs_fallocate) 8616e22ab9dSJiri Olsa BTF_ID(func, dentry_open) 8626e22ab9dSJiri Olsa BTF_ID(func, vfs_getattr) 8636e22ab9dSJiri Olsa BTF_ID(func, filp_close) 8646e22ab9dSJiri Olsa BTF_SET_END(btf_allowlist_d_path) 8656e22ab9dSJiri Olsa 8666e22ab9dSJiri Olsa static bool bpf_d_path_allowed(const struct bpf_prog *prog) 8676e22ab9dSJiri Olsa { 8683d06f34aSSong Liu if (prog->type == BPF_PROG_TYPE_TRACING && 8693d06f34aSSong Liu prog->expected_attach_type == BPF_TRACE_ITER) 8703d06f34aSSong Liu return true; 8713d06f34aSSong Liu 8726f100640SKP Singh if (prog->type == BPF_PROG_TYPE_LSM) 8736f100640SKP Singh return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id); 8746f100640SKP Singh 8756f100640SKP Singh return btf_id_set_contains(&btf_allowlist_d_path, 8766f100640SKP Singh prog->aux->attach_btf_id); 8776e22ab9dSJiri Olsa } 8786e22ab9dSJiri Olsa 8799436ef6eSLorenz Bauer BTF_ID_LIST_SINGLE(bpf_d_path_btf_ids, struct, path) 8806e22ab9dSJiri Olsa 8816e22ab9dSJiri Olsa static const struct bpf_func_proto bpf_d_path_proto = { 8826e22ab9dSJiri Olsa .func = bpf_d_path, 8836e22ab9dSJiri Olsa .gpl_only = false, 8846e22ab9dSJiri Olsa .ret_type = RET_INTEGER, 8856e22ab9dSJiri Olsa .arg1_type = ARG_PTR_TO_BTF_ID, 8869436ef6eSLorenz Bauer .arg1_btf_id = &bpf_d_path_btf_ids[0], 8876e22ab9dSJiri Olsa .arg2_type = ARG_PTR_TO_MEM, 8886e22ab9dSJiri Olsa .arg3_type = ARG_CONST_SIZE_OR_ZERO, 8896e22ab9dSJiri Olsa .allowed = bpf_d_path_allowed, 8906e22ab9dSJiri Olsa }; 8916e22ab9dSJiri Olsa 892c4d0bfb4SAlan Maguire #define BTF_F_ALL (BTF_F_COMPACT | BTF_F_NONAME | \ 893c4d0bfb4SAlan Maguire BTF_F_PTR_RAW | BTF_F_ZERO) 894c4d0bfb4SAlan Maguire 895c4d0bfb4SAlan Maguire static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size, 896c4d0bfb4SAlan Maguire u64 flags, const struct btf **btf, 897c4d0bfb4SAlan Maguire s32 *btf_id) 898c4d0bfb4SAlan Maguire { 899c4d0bfb4SAlan Maguire const struct btf_type *t; 900c4d0bfb4SAlan Maguire 901c4d0bfb4SAlan Maguire if (unlikely(flags & ~(BTF_F_ALL))) 902c4d0bfb4SAlan Maguire return -EINVAL; 903c4d0bfb4SAlan Maguire 904c4d0bfb4SAlan Maguire if (btf_ptr_size != sizeof(struct btf_ptr)) 905c4d0bfb4SAlan Maguire return -EINVAL; 906c4d0bfb4SAlan Maguire 907c4d0bfb4SAlan Maguire *btf = bpf_get_btf_vmlinux(); 908c4d0bfb4SAlan Maguire 909c4d0bfb4SAlan Maguire if (IS_ERR_OR_NULL(*btf)) 910abbaa433SWang Qing return IS_ERR(*btf) ? PTR_ERR(*btf) : -EINVAL; 911c4d0bfb4SAlan Maguire 912c4d0bfb4SAlan Maguire if (ptr->type_id > 0) 913c4d0bfb4SAlan Maguire *btf_id = ptr->type_id; 914c4d0bfb4SAlan Maguire else 915c4d0bfb4SAlan Maguire return -EINVAL; 916c4d0bfb4SAlan Maguire 917c4d0bfb4SAlan Maguire if (*btf_id > 0) 918c4d0bfb4SAlan Maguire t = btf_type_by_id(*btf, *btf_id); 919c4d0bfb4SAlan Maguire if (*btf_id <= 0 || !t) 920c4d0bfb4SAlan Maguire return -ENOENT; 921c4d0bfb4SAlan Maguire 922c4d0bfb4SAlan Maguire return 0; 923c4d0bfb4SAlan Maguire } 924c4d0bfb4SAlan Maguire 925c4d0bfb4SAlan Maguire BPF_CALL_5(bpf_snprintf_btf, char *, str, u32, str_size, struct btf_ptr *, ptr, 926c4d0bfb4SAlan Maguire u32, btf_ptr_size, u64, flags) 927c4d0bfb4SAlan Maguire { 928c4d0bfb4SAlan Maguire const struct btf *btf; 929c4d0bfb4SAlan Maguire s32 btf_id; 930c4d0bfb4SAlan Maguire int ret; 931c4d0bfb4SAlan Maguire 932c4d0bfb4SAlan Maguire ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id); 933c4d0bfb4SAlan Maguire if (ret) 934c4d0bfb4SAlan Maguire return ret; 935c4d0bfb4SAlan Maguire 936c4d0bfb4SAlan Maguire return btf_type_snprintf_show(btf, btf_id, ptr->ptr, str, str_size, 937c4d0bfb4SAlan Maguire flags); 938c4d0bfb4SAlan Maguire } 939c4d0bfb4SAlan Maguire 940c4d0bfb4SAlan Maguire const struct bpf_func_proto bpf_snprintf_btf_proto = { 941c4d0bfb4SAlan Maguire .func = bpf_snprintf_btf, 942c4d0bfb4SAlan Maguire .gpl_only = false, 943c4d0bfb4SAlan Maguire .ret_type = RET_INTEGER, 944c4d0bfb4SAlan Maguire .arg1_type = ARG_PTR_TO_MEM, 945c4d0bfb4SAlan Maguire .arg2_type = ARG_CONST_SIZE, 946c4d0bfb4SAlan Maguire .arg3_type = ARG_PTR_TO_MEM, 947c4d0bfb4SAlan Maguire .arg4_type = ARG_CONST_SIZE, 948c4d0bfb4SAlan Maguire .arg5_type = ARG_ANYTHING, 949c4d0bfb4SAlan Maguire }; 950c4d0bfb4SAlan Maguire 951fc611f47SKP Singh const struct bpf_func_proto * 952fc611f47SKP Singh bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 9532541517cSAlexei Starovoitov { 9542541517cSAlexei Starovoitov switch (func_id) { 9552541517cSAlexei Starovoitov case BPF_FUNC_map_lookup_elem: 9562541517cSAlexei Starovoitov return &bpf_map_lookup_elem_proto; 9572541517cSAlexei Starovoitov case BPF_FUNC_map_update_elem: 9582541517cSAlexei Starovoitov return &bpf_map_update_elem_proto; 9592541517cSAlexei Starovoitov case BPF_FUNC_map_delete_elem: 9602541517cSAlexei Starovoitov return &bpf_map_delete_elem_proto; 96102a8c817SAlban Crequy case BPF_FUNC_map_push_elem: 96202a8c817SAlban Crequy return &bpf_map_push_elem_proto; 96302a8c817SAlban Crequy case BPF_FUNC_map_pop_elem: 96402a8c817SAlban Crequy return &bpf_map_pop_elem_proto; 96502a8c817SAlban Crequy case BPF_FUNC_map_peek_elem: 96602a8c817SAlban Crequy return &bpf_map_peek_elem_proto; 967d9847d31SAlexei Starovoitov case BPF_FUNC_ktime_get_ns: 968d9847d31SAlexei Starovoitov return &bpf_ktime_get_ns_proto; 96971d19214SMaciej Żenczykowski case BPF_FUNC_ktime_get_boot_ns: 97071d19214SMaciej Żenczykowski return &bpf_ktime_get_boot_ns_proto; 971d0551261SDmitrii Banshchikov case BPF_FUNC_ktime_get_coarse_ns: 972d0551261SDmitrii Banshchikov return &bpf_ktime_get_coarse_ns_proto; 97304fd61abSAlexei Starovoitov case BPF_FUNC_tail_call: 97404fd61abSAlexei Starovoitov return &bpf_tail_call_proto; 975ffeedafbSAlexei Starovoitov case BPF_FUNC_get_current_pid_tgid: 976ffeedafbSAlexei Starovoitov return &bpf_get_current_pid_tgid_proto; 977606274c5SAlexei Starovoitov case BPF_FUNC_get_current_task: 978606274c5SAlexei Starovoitov return &bpf_get_current_task_proto; 9793ca1032aSKP Singh case BPF_FUNC_get_current_task_btf: 9803ca1032aSKP Singh return &bpf_get_current_task_btf_proto; 981ffeedafbSAlexei Starovoitov case BPF_FUNC_get_current_uid_gid: 982ffeedafbSAlexei Starovoitov return &bpf_get_current_uid_gid_proto; 983ffeedafbSAlexei Starovoitov case BPF_FUNC_get_current_comm: 984ffeedafbSAlexei Starovoitov return &bpf_get_current_comm_proto; 9859c959c86SAlexei Starovoitov case BPF_FUNC_trace_printk: 9860756ea3eSAlexei Starovoitov return bpf_get_trace_printk_proto(); 987ab1973d3SAlexei Starovoitov case BPF_FUNC_get_smp_processor_id: 988ab1973d3SAlexei Starovoitov return &bpf_get_smp_processor_id_proto; 9892d0e30c3SDaniel Borkmann case BPF_FUNC_get_numa_node_id: 9902d0e30c3SDaniel Borkmann return &bpf_get_numa_node_id_proto; 99135578d79SKaixu Xia case BPF_FUNC_perf_event_read: 99235578d79SKaixu Xia return &bpf_perf_event_read_proto; 99396ae5227SSargun Dhillon case BPF_FUNC_probe_write_user: 99496ae5227SSargun Dhillon return bpf_get_probe_write_proto(); 99560d20f91SSargun Dhillon case BPF_FUNC_current_task_under_cgroup: 99660d20f91SSargun Dhillon return &bpf_current_task_under_cgroup_proto; 9978937bd80SAlexei Starovoitov case BPF_FUNC_get_prandom_u32: 9988937bd80SAlexei Starovoitov return &bpf_get_prandom_u32_proto; 9996ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read_user: 10006ae08ae3SDaniel Borkmann return &bpf_probe_read_user_proto; 10016ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read_kernel: 1002*ff40e510SDaniel Borkmann return security_locked_down(LOCKDOWN_BPF_READ) < 0 ? 1003*ff40e510SDaniel Borkmann NULL : &bpf_probe_read_kernel_proto; 10046ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read_user_str: 10056ae08ae3SDaniel Borkmann return &bpf_probe_read_user_str_proto; 10066ae08ae3SDaniel Borkmann case BPF_FUNC_probe_read_kernel_str: 1007*ff40e510SDaniel Borkmann return security_locked_down(LOCKDOWN_BPF_READ) < 0 ? 1008*ff40e510SDaniel Borkmann NULL : &bpf_probe_read_kernel_str_proto; 10090ebeea8cSDaniel Borkmann #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE 10100ebeea8cSDaniel Borkmann case BPF_FUNC_probe_read: 1011*ff40e510SDaniel Borkmann return security_locked_down(LOCKDOWN_BPF_READ) < 0 ? 1012*ff40e510SDaniel Borkmann NULL : &bpf_probe_read_compat_proto; 1013a5e8c070SGianluca Borello case BPF_FUNC_probe_read_str: 1014*ff40e510SDaniel Borkmann return security_locked_down(LOCKDOWN_BPF_READ) < 0 ? 1015*ff40e510SDaniel Borkmann NULL : &bpf_probe_read_compat_str_proto; 10160ebeea8cSDaniel Borkmann #endif 101734ea38caSYonghong Song #ifdef CONFIG_CGROUPS 1018bf6fa2c8SYonghong Song case BPF_FUNC_get_current_cgroup_id: 1019bf6fa2c8SYonghong Song return &bpf_get_current_cgroup_id_proto; 102034ea38caSYonghong Song #endif 10218b401f9eSYonghong Song case BPF_FUNC_send_signal: 10228b401f9eSYonghong Song return &bpf_send_signal_proto; 10238482941fSYonghong Song case BPF_FUNC_send_signal_thread: 10248482941fSYonghong Song return &bpf_send_signal_thread_proto; 1025b80b033bSSong Liu case BPF_FUNC_perf_event_read_value: 1026b80b033bSSong Liu return &bpf_perf_event_read_value_proto; 1027b4490c5cSCarlos Neira case BPF_FUNC_get_ns_current_pid_tgid: 1028b4490c5cSCarlos Neira return &bpf_get_ns_current_pid_tgid_proto; 1029457f4436SAndrii Nakryiko case BPF_FUNC_ringbuf_output: 1030457f4436SAndrii Nakryiko return &bpf_ringbuf_output_proto; 1031457f4436SAndrii Nakryiko case BPF_FUNC_ringbuf_reserve: 1032457f4436SAndrii Nakryiko return &bpf_ringbuf_reserve_proto; 1033457f4436SAndrii Nakryiko case BPF_FUNC_ringbuf_submit: 1034457f4436SAndrii Nakryiko return &bpf_ringbuf_submit_proto; 1035457f4436SAndrii Nakryiko case BPF_FUNC_ringbuf_discard: 1036457f4436SAndrii Nakryiko return &bpf_ringbuf_discard_proto; 1037457f4436SAndrii Nakryiko case BPF_FUNC_ringbuf_query: 1038457f4436SAndrii Nakryiko return &bpf_ringbuf_query_proto; 103972e2b2b6SYonghong Song case BPF_FUNC_jiffies64: 104072e2b2b6SYonghong Song return &bpf_jiffies64_proto; 1041fa28dcb8SSong Liu case BPF_FUNC_get_task_stack: 1042fa28dcb8SSong Liu return &bpf_get_task_stack_proto; 104307be4c4aSAlexei Starovoitov case BPF_FUNC_copy_from_user: 104407be4c4aSAlexei Starovoitov return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL; 1045c4d0bfb4SAlan Maguire case BPF_FUNC_snprintf_btf: 1046c4d0bfb4SAlan Maguire return &bpf_snprintf_btf_proto; 1047b7906b70SAndrii Nakryiko case BPF_FUNC_per_cpu_ptr: 1048eaa6bcb7SHao Luo return &bpf_per_cpu_ptr_proto; 1049b7906b70SAndrii Nakryiko case BPF_FUNC_this_cpu_ptr: 105063d9b80dSHao Luo return &bpf_this_cpu_ptr_proto; 1051a10787e6SSong Liu case BPF_FUNC_task_storage_get: 1052a10787e6SSong Liu return &bpf_task_storage_get_proto; 1053a10787e6SSong Liu case BPF_FUNC_task_storage_delete: 1054a10787e6SSong Liu return &bpf_task_storage_delete_proto; 105569c087baSYonghong Song case BPF_FUNC_for_each_map_elem: 105669c087baSYonghong Song return &bpf_for_each_map_elem_proto; 10577b15523aSFlorent Revest case BPF_FUNC_snprintf: 10587b15523aSFlorent Revest return &bpf_snprintf_proto; 10599fd82b61SAlexei Starovoitov default: 10609fd82b61SAlexei Starovoitov return NULL; 10619fd82b61SAlexei Starovoitov } 10629fd82b61SAlexei Starovoitov } 10639fd82b61SAlexei Starovoitov 10645e43f899SAndrey Ignatov static const struct bpf_func_proto * 10655e43f899SAndrey Ignatov kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 10669fd82b61SAlexei Starovoitov { 10679fd82b61SAlexei Starovoitov switch (func_id) { 1068a43eec30SAlexei Starovoitov case BPF_FUNC_perf_event_output: 1069a43eec30SAlexei Starovoitov return &bpf_perf_event_output_proto; 1070d5a3b1f6SAlexei Starovoitov case BPF_FUNC_get_stackid: 1071d5a3b1f6SAlexei Starovoitov return &bpf_get_stackid_proto; 1072c195651eSYonghong Song case BPF_FUNC_get_stack: 1073c195651eSYonghong Song return &bpf_get_stack_proto; 10749802d865SJosef Bacik #ifdef CONFIG_BPF_KPROBE_OVERRIDE 10759802d865SJosef Bacik case BPF_FUNC_override_return: 10769802d865SJosef Bacik return &bpf_override_return_proto; 10779802d865SJosef Bacik #endif 10782541517cSAlexei Starovoitov default: 1079fc611f47SKP Singh return bpf_tracing_func_proto(func_id, prog); 10802541517cSAlexei Starovoitov } 10812541517cSAlexei Starovoitov } 10822541517cSAlexei Starovoitov 10832541517cSAlexei Starovoitov /* bpf+kprobe programs can access fields of 'struct pt_regs' */ 108419de99f7SAlexei Starovoitov static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type type, 10855e43f899SAndrey Ignatov const struct bpf_prog *prog, 108623994631SYonghong Song struct bpf_insn_access_aux *info) 10872541517cSAlexei Starovoitov { 10882541517cSAlexei Starovoitov if (off < 0 || off >= sizeof(struct pt_regs)) 10892541517cSAlexei Starovoitov return false; 10902541517cSAlexei Starovoitov if (type != BPF_READ) 10912541517cSAlexei Starovoitov return false; 10922541517cSAlexei Starovoitov if (off % size != 0) 10932541517cSAlexei Starovoitov return false; 10942d071c64SDaniel Borkmann /* 10952d071c64SDaniel Borkmann * Assertion for 32 bit to make sure last 8 byte access 10962d071c64SDaniel Borkmann * (BPF_DW) to the last 4 byte member is disallowed. 10972d071c64SDaniel Borkmann */ 10982d071c64SDaniel Borkmann if (off + size > sizeof(struct pt_regs)) 10992d071c64SDaniel Borkmann return false; 11002d071c64SDaniel Borkmann 11012541517cSAlexei Starovoitov return true; 11022541517cSAlexei Starovoitov } 11032541517cSAlexei Starovoitov 11047de16e3aSJakub Kicinski const struct bpf_verifier_ops kprobe_verifier_ops = { 11052541517cSAlexei Starovoitov .get_func_proto = kprobe_prog_func_proto, 11062541517cSAlexei Starovoitov .is_valid_access = kprobe_prog_is_valid_access, 11072541517cSAlexei Starovoitov }; 11082541517cSAlexei Starovoitov 11097de16e3aSJakub Kicinski const struct bpf_prog_ops kprobe_prog_ops = { 11107de16e3aSJakub Kicinski }; 11117de16e3aSJakub Kicinski 1112f3694e00SDaniel Borkmann BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map, 1113f3694e00SDaniel Borkmann u64, flags, void *, data, u64, size) 11149940d67cSAlexei Starovoitov { 1115f3694e00SDaniel Borkmann struct pt_regs *regs = *(struct pt_regs **)tp_buff; 1116f3694e00SDaniel Borkmann 11179940d67cSAlexei Starovoitov /* 11189940d67cSAlexei Starovoitov * r1 points to perf tracepoint buffer where first 8 bytes are hidden 11199940d67cSAlexei Starovoitov * from bpf program and contain a pointer to 'struct pt_regs'. Fetch it 1120f3694e00SDaniel Borkmann * from there and call the same bpf_perf_event_output() helper inline. 11219940d67cSAlexei Starovoitov */ 1122f3694e00SDaniel Borkmann return ____bpf_perf_event_output(regs, map, flags, data, size); 11239940d67cSAlexei Starovoitov } 11249940d67cSAlexei Starovoitov 11259940d67cSAlexei Starovoitov static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { 11269940d67cSAlexei Starovoitov .func = bpf_perf_event_output_tp, 11279940d67cSAlexei Starovoitov .gpl_only = true, 11289940d67cSAlexei Starovoitov .ret_type = RET_INTEGER, 11299940d67cSAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 11309940d67cSAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 11319940d67cSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 113239f19ebbSAlexei Starovoitov .arg4_type = ARG_PTR_TO_MEM, 1133a60dd35dSGianluca Borello .arg5_type = ARG_CONST_SIZE_OR_ZERO, 11349940d67cSAlexei Starovoitov }; 11359940d67cSAlexei Starovoitov 1136f3694e00SDaniel Borkmann BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map, 1137f3694e00SDaniel Borkmann u64, flags) 11389940d67cSAlexei Starovoitov { 1139f3694e00SDaniel Borkmann struct pt_regs *regs = *(struct pt_regs **)tp_buff; 11409940d67cSAlexei Starovoitov 1141f3694e00SDaniel Borkmann /* 1142f3694e00SDaniel Borkmann * Same comment as in bpf_perf_event_output_tp(), only that this time 1143f3694e00SDaniel Borkmann * the other helper's function body cannot be inlined due to being 1144f3694e00SDaniel Borkmann * external, thus we need to call raw helper function. 1145f3694e00SDaniel Borkmann */ 1146f3694e00SDaniel Borkmann return bpf_get_stackid((unsigned long) regs, (unsigned long) map, 1147f3694e00SDaniel Borkmann flags, 0, 0); 11489940d67cSAlexei Starovoitov } 11499940d67cSAlexei Starovoitov 11509940d67cSAlexei Starovoitov static const struct bpf_func_proto bpf_get_stackid_proto_tp = { 11519940d67cSAlexei Starovoitov .func = bpf_get_stackid_tp, 11529940d67cSAlexei Starovoitov .gpl_only = true, 11539940d67cSAlexei Starovoitov .ret_type = RET_INTEGER, 11549940d67cSAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 11559940d67cSAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 11569940d67cSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 11579940d67cSAlexei Starovoitov }; 11589940d67cSAlexei Starovoitov 1159c195651eSYonghong Song BPF_CALL_4(bpf_get_stack_tp, void *, tp_buff, void *, buf, u32, size, 1160c195651eSYonghong Song u64, flags) 1161c195651eSYonghong Song { 1162c195651eSYonghong Song struct pt_regs *regs = *(struct pt_regs **)tp_buff; 1163c195651eSYonghong Song 1164c195651eSYonghong Song return bpf_get_stack((unsigned long) regs, (unsigned long) buf, 1165c195651eSYonghong Song (unsigned long) size, flags, 0); 1166c195651eSYonghong Song } 1167c195651eSYonghong Song 1168c195651eSYonghong Song static const struct bpf_func_proto bpf_get_stack_proto_tp = { 1169c195651eSYonghong Song .func = bpf_get_stack_tp, 1170c195651eSYonghong Song .gpl_only = true, 1171c195651eSYonghong Song .ret_type = RET_INTEGER, 1172c195651eSYonghong Song .arg1_type = ARG_PTR_TO_CTX, 1173c195651eSYonghong Song .arg2_type = ARG_PTR_TO_UNINIT_MEM, 1174c195651eSYonghong Song .arg3_type = ARG_CONST_SIZE_OR_ZERO, 1175c195651eSYonghong Song .arg4_type = ARG_ANYTHING, 1176c195651eSYonghong Song }; 1177c195651eSYonghong Song 11785e43f899SAndrey Ignatov static const struct bpf_func_proto * 11795e43f899SAndrey Ignatov tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 11809fd82b61SAlexei Starovoitov { 11819fd82b61SAlexei Starovoitov switch (func_id) { 11829fd82b61SAlexei Starovoitov case BPF_FUNC_perf_event_output: 11839940d67cSAlexei Starovoitov return &bpf_perf_event_output_proto_tp; 11849fd82b61SAlexei Starovoitov case BPF_FUNC_get_stackid: 11859940d67cSAlexei Starovoitov return &bpf_get_stackid_proto_tp; 1186c195651eSYonghong Song case BPF_FUNC_get_stack: 1187c195651eSYonghong Song return &bpf_get_stack_proto_tp; 11889fd82b61SAlexei Starovoitov default: 1189fc611f47SKP Singh return bpf_tracing_func_proto(func_id, prog); 11909fd82b61SAlexei Starovoitov } 11919fd82b61SAlexei Starovoitov } 11929fd82b61SAlexei Starovoitov 119319de99f7SAlexei Starovoitov static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type, 11945e43f899SAndrey Ignatov const struct bpf_prog *prog, 119523994631SYonghong Song struct bpf_insn_access_aux *info) 11969fd82b61SAlexei Starovoitov { 11979fd82b61SAlexei Starovoitov if (off < sizeof(void *) || off >= PERF_MAX_TRACE_SIZE) 11989fd82b61SAlexei Starovoitov return false; 11999fd82b61SAlexei Starovoitov if (type != BPF_READ) 12009fd82b61SAlexei Starovoitov return false; 12019fd82b61SAlexei Starovoitov if (off % size != 0) 12029fd82b61SAlexei Starovoitov return false; 12032d071c64SDaniel Borkmann 12042d071c64SDaniel Borkmann BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(__u64)); 12059fd82b61SAlexei Starovoitov return true; 12069fd82b61SAlexei Starovoitov } 12079fd82b61SAlexei Starovoitov 12087de16e3aSJakub Kicinski const struct bpf_verifier_ops tracepoint_verifier_ops = { 12099fd82b61SAlexei Starovoitov .get_func_proto = tp_prog_func_proto, 12109fd82b61SAlexei Starovoitov .is_valid_access = tp_prog_is_valid_access, 12119fd82b61SAlexei Starovoitov }; 12129fd82b61SAlexei Starovoitov 12137de16e3aSJakub Kicinski const struct bpf_prog_ops tracepoint_prog_ops = { 12147de16e3aSJakub Kicinski }; 12157de16e3aSJakub Kicinski 1216f005afedSYonghong Song BPF_CALL_3(bpf_perf_prog_read_value, struct bpf_perf_event_data_kern *, ctx, 1217f005afedSYonghong Song struct bpf_perf_event_value *, buf, u32, size) 1218f005afedSYonghong Song { 1219f005afedSYonghong Song int err = -EINVAL; 1220f005afedSYonghong Song 1221f005afedSYonghong Song if (unlikely(size != sizeof(struct bpf_perf_event_value))) 1222f005afedSYonghong Song goto clear; 1223f005afedSYonghong Song err = perf_event_read_local(ctx->event, &buf->counter, &buf->enabled, 1224f005afedSYonghong Song &buf->running); 1225f005afedSYonghong Song if (unlikely(err)) 1226f005afedSYonghong Song goto clear; 1227f005afedSYonghong Song return 0; 1228f005afedSYonghong Song clear: 1229f005afedSYonghong Song memset(buf, 0, size); 1230f005afedSYonghong Song return err; 1231f005afedSYonghong Song } 1232f005afedSYonghong Song 1233f005afedSYonghong Song static const struct bpf_func_proto bpf_perf_prog_read_value_proto = { 1234f005afedSYonghong Song .func = bpf_perf_prog_read_value, 1235f005afedSYonghong Song .gpl_only = true, 1236f005afedSYonghong Song .ret_type = RET_INTEGER, 1237f005afedSYonghong Song .arg1_type = ARG_PTR_TO_CTX, 1238f005afedSYonghong Song .arg2_type = ARG_PTR_TO_UNINIT_MEM, 1239f005afedSYonghong Song .arg3_type = ARG_CONST_SIZE, 1240f005afedSYonghong Song }; 1241f005afedSYonghong Song 1242fff7b643SDaniel Xu BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx, 1243fff7b643SDaniel Xu void *, buf, u32, size, u64, flags) 1244fff7b643SDaniel Xu { 1245fff7b643SDaniel Xu #ifndef CONFIG_X86 1246fff7b643SDaniel Xu return -ENOENT; 1247fff7b643SDaniel Xu #else 1248fff7b643SDaniel Xu static const u32 br_entry_size = sizeof(struct perf_branch_entry); 1249fff7b643SDaniel Xu struct perf_branch_stack *br_stack = ctx->data->br_stack; 1250fff7b643SDaniel Xu u32 to_copy; 1251fff7b643SDaniel Xu 1252fff7b643SDaniel Xu if (unlikely(flags & ~BPF_F_GET_BRANCH_RECORDS_SIZE)) 1253fff7b643SDaniel Xu return -EINVAL; 1254fff7b643SDaniel Xu 1255fff7b643SDaniel Xu if (unlikely(!br_stack)) 1256fff7b643SDaniel Xu return -EINVAL; 1257fff7b643SDaniel Xu 1258fff7b643SDaniel Xu if (flags & BPF_F_GET_BRANCH_RECORDS_SIZE) 1259fff7b643SDaniel Xu return br_stack->nr * br_entry_size; 1260fff7b643SDaniel Xu 1261fff7b643SDaniel Xu if (!buf || (size % br_entry_size != 0)) 1262fff7b643SDaniel Xu return -EINVAL; 1263fff7b643SDaniel Xu 1264fff7b643SDaniel Xu to_copy = min_t(u32, br_stack->nr * br_entry_size, size); 1265fff7b643SDaniel Xu memcpy(buf, br_stack->entries, to_copy); 1266fff7b643SDaniel Xu 1267fff7b643SDaniel Xu return to_copy; 1268fff7b643SDaniel Xu #endif 1269fff7b643SDaniel Xu } 1270fff7b643SDaniel Xu 1271fff7b643SDaniel Xu static const struct bpf_func_proto bpf_read_branch_records_proto = { 1272fff7b643SDaniel Xu .func = bpf_read_branch_records, 1273fff7b643SDaniel Xu .gpl_only = true, 1274fff7b643SDaniel Xu .ret_type = RET_INTEGER, 1275fff7b643SDaniel Xu .arg1_type = ARG_PTR_TO_CTX, 1276fff7b643SDaniel Xu .arg2_type = ARG_PTR_TO_MEM_OR_NULL, 1277fff7b643SDaniel Xu .arg3_type = ARG_CONST_SIZE_OR_ZERO, 1278fff7b643SDaniel Xu .arg4_type = ARG_ANYTHING, 1279fff7b643SDaniel Xu }; 1280fff7b643SDaniel Xu 12815e43f899SAndrey Ignatov static const struct bpf_func_proto * 12825e43f899SAndrey Ignatov pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1283f005afedSYonghong Song { 1284f005afedSYonghong Song switch (func_id) { 1285f005afedSYonghong Song case BPF_FUNC_perf_event_output: 1286f005afedSYonghong Song return &bpf_perf_event_output_proto_tp; 1287f005afedSYonghong Song case BPF_FUNC_get_stackid: 12887b04d6d6SSong Liu return &bpf_get_stackid_proto_pe; 1289c195651eSYonghong Song case BPF_FUNC_get_stack: 12907b04d6d6SSong Liu return &bpf_get_stack_proto_pe; 1291f005afedSYonghong Song case BPF_FUNC_perf_prog_read_value: 1292f005afedSYonghong Song return &bpf_perf_prog_read_value_proto; 1293fff7b643SDaniel Xu case BPF_FUNC_read_branch_records: 1294fff7b643SDaniel Xu return &bpf_read_branch_records_proto; 1295f005afedSYonghong Song default: 1296fc611f47SKP Singh return bpf_tracing_func_proto(func_id, prog); 1297f005afedSYonghong Song } 1298f005afedSYonghong Song } 1299f005afedSYonghong Song 1300c4f6699dSAlexei Starovoitov /* 1301c4f6699dSAlexei Starovoitov * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp 1302c4f6699dSAlexei Starovoitov * to avoid potential recursive reuse issue when/if tracepoints are added 13039594dc3cSMatt Mullins * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack. 13049594dc3cSMatt Mullins * 13059594dc3cSMatt Mullins * Since raw tracepoints run despite bpf_prog_active, support concurrent usage 13069594dc3cSMatt Mullins * in normal, irq, and nmi context. 1307c4f6699dSAlexei Starovoitov */ 13089594dc3cSMatt Mullins struct bpf_raw_tp_regs { 13099594dc3cSMatt Mullins struct pt_regs regs[3]; 13109594dc3cSMatt Mullins }; 13119594dc3cSMatt Mullins static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs); 13129594dc3cSMatt Mullins static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level); 13139594dc3cSMatt Mullins static struct pt_regs *get_bpf_raw_tp_regs(void) 13149594dc3cSMatt Mullins { 13159594dc3cSMatt Mullins struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs); 13169594dc3cSMatt Mullins int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level); 13179594dc3cSMatt Mullins 13189594dc3cSMatt Mullins if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(tp_regs->regs))) { 13199594dc3cSMatt Mullins this_cpu_dec(bpf_raw_tp_nest_level); 13209594dc3cSMatt Mullins return ERR_PTR(-EBUSY); 13219594dc3cSMatt Mullins } 13229594dc3cSMatt Mullins 13239594dc3cSMatt Mullins return &tp_regs->regs[nest_level - 1]; 13249594dc3cSMatt Mullins } 13259594dc3cSMatt Mullins 13269594dc3cSMatt Mullins static void put_bpf_raw_tp_regs(void) 13279594dc3cSMatt Mullins { 13289594dc3cSMatt Mullins this_cpu_dec(bpf_raw_tp_nest_level); 13299594dc3cSMatt Mullins } 13309594dc3cSMatt Mullins 1331c4f6699dSAlexei Starovoitov BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args, 1332c4f6699dSAlexei Starovoitov struct bpf_map *, map, u64, flags, void *, data, u64, size) 1333c4f6699dSAlexei Starovoitov { 13349594dc3cSMatt Mullins struct pt_regs *regs = get_bpf_raw_tp_regs(); 13359594dc3cSMatt Mullins int ret; 13369594dc3cSMatt Mullins 13379594dc3cSMatt Mullins if (IS_ERR(regs)) 13389594dc3cSMatt Mullins return PTR_ERR(regs); 1339c4f6699dSAlexei Starovoitov 1340c4f6699dSAlexei Starovoitov perf_fetch_caller_regs(regs); 13419594dc3cSMatt Mullins ret = ____bpf_perf_event_output(regs, map, flags, data, size); 13429594dc3cSMatt Mullins 13439594dc3cSMatt Mullins put_bpf_raw_tp_regs(); 13449594dc3cSMatt Mullins return ret; 1345c4f6699dSAlexei Starovoitov } 1346c4f6699dSAlexei Starovoitov 1347c4f6699dSAlexei Starovoitov static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { 1348c4f6699dSAlexei Starovoitov .func = bpf_perf_event_output_raw_tp, 1349c4f6699dSAlexei Starovoitov .gpl_only = true, 1350c4f6699dSAlexei Starovoitov .ret_type = RET_INTEGER, 1351c4f6699dSAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 1352c4f6699dSAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 1353c4f6699dSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 1354c4f6699dSAlexei Starovoitov .arg4_type = ARG_PTR_TO_MEM, 1355c4f6699dSAlexei Starovoitov .arg5_type = ARG_CONST_SIZE_OR_ZERO, 1356c4f6699dSAlexei Starovoitov }; 1357c4f6699dSAlexei Starovoitov 1358a7658e1aSAlexei Starovoitov extern const struct bpf_func_proto bpf_skb_output_proto; 1359d831ee84SEelco Chaudron extern const struct bpf_func_proto bpf_xdp_output_proto; 1360a7658e1aSAlexei Starovoitov 1361c4f6699dSAlexei Starovoitov BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args, 1362c4f6699dSAlexei Starovoitov struct bpf_map *, map, u64, flags) 1363c4f6699dSAlexei Starovoitov { 13649594dc3cSMatt Mullins struct pt_regs *regs = get_bpf_raw_tp_regs(); 13659594dc3cSMatt Mullins int ret; 13669594dc3cSMatt Mullins 13679594dc3cSMatt Mullins if (IS_ERR(regs)) 13689594dc3cSMatt Mullins return PTR_ERR(regs); 1369c4f6699dSAlexei Starovoitov 1370c4f6699dSAlexei Starovoitov perf_fetch_caller_regs(regs); 1371c4f6699dSAlexei Starovoitov /* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */ 13729594dc3cSMatt Mullins ret = bpf_get_stackid((unsigned long) regs, (unsigned long) map, 1373c4f6699dSAlexei Starovoitov flags, 0, 0); 13749594dc3cSMatt Mullins put_bpf_raw_tp_regs(); 13759594dc3cSMatt Mullins return ret; 1376c4f6699dSAlexei Starovoitov } 1377c4f6699dSAlexei Starovoitov 1378c4f6699dSAlexei Starovoitov static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = { 1379c4f6699dSAlexei Starovoitov .func = bpf_get_stackid_raw_tp, 1380c4f6699dSAlexei Starovoitov .gpl_only = true, 1381c4f6699dSAlexei Starovoitov .ret_type = RET_INTEGER, 1382c4f6699dSAlexei Starovoitov .arg1_type = ARG_PTR_TO_CTX, 1383c4f6699dSAlexei Starovoitov .arg2_type = ARG_CONST_MAP_PTR, 1384c4f6699dSAlexei Starovoitov .arg3_type = ARG_ANYTHING, 1385c4f6699dSAlexei Starovoitov }; 1386c4f6699dSAlexei Starovoitov 1387c195651eSYonghong Song BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args, 1388c195651eSYonghong Song void *, buf, u32, size, u64, flags) 1389c195651eSYonghong Song { 13909594dc3cSMatt Mullins struct pt_regs *regs = get_bpf_raw_tp_regs(); 13919594dc3cSMatt Mullins int ret; 13929594dc3cSMatt Mullins 13939594dc3cSMatt Mullins if (IS_ERR(regs)) 13949594dc3cSMatt Mullins return PTR_ERR(regs); 1395c195651eSYonghong Song 1396c195651eSYonghong Song perf_fetch_caller_regs(regs); 13979594dc3cSMatt Mullins ret = bpf_get_stack((unsigned long) regs, (unsigned long) buf, 1398c195651eSYonghong Song (unsigned long) size, flags, 0); 13999594dc3cSMatt Mullins put_bpf_raw_tp_regs(); 14009594dc3cSMatt Mullins return ret; 1401c195651eSYonghong Song } 1402c195651eSYonghong Song 1403c195651eSYonghong Song static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { 1404c195651eSYonghong Song .func = bpf_get_stack_raw_tp, 1405c195651eSYonghong Song .gpl_only = true, 1406c195651eSYonghong Song .ret_type = RET_INTEGER, 1407c195651eSYonghong Song .arg1_type = ARG_PTR_TO_CTX, 1408c195651eSYonghong Song .arg2_type = ARG_PTR_TO_MEM, 1409c195651eSYonghong Song .arg3_type = ARG_CONST_SIZE_OR_ZERO, 1410c195651eSYonghong Song .arg4_type = ARG_ANYTHING, 1411c195651eSYonghong Song }; 1412c195651eSYonghong Song 14135e43f899SAndrey Ignatov static const struct bpf_func_proto * 14145e43f899SAndrey Ignatov raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1415c4f6699dSAlexei Starovoitov { 1416c4f6699dSAlexei Starovoitov switch (func_id) { 1417c4f6699dSAlexei Starovoitov case BPF_FUNC_perf_event_output: 1418c4f6699dSAlexei Starovoitov return &bpf_perf_event_output_proto_raw_tp; 1419c4f6699dSAlexei Starovoitov case BPF_FUNC_get_stackid: 1420c4f6699dSAlexei Starovoitov return &bpf_get_stackid_proto_raw_tp; 1421c195651eSYonghong Song case BPF_FUNC_get_stack: 1422c195651eSYonghong Song return &bpf_get_stack_proto_raw_tp; 1423c4f6699dSAlexei Starovoitov default: 1424fc611f47SKP Singh return bpf_tracing_func_proto(func_id, prog); 1425c4f6699dSAlexei Starovoitov } 1426c4f6699dSAlexei Starovoitov } 1427c4f6699dSAlexei Starovoitov 1428958a3f2dSJiri Olsa const struct bpf_func_proto * 1429f1b9509cSAlexei Starovoitov tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1430f1b9509cSAlexei Starovoitov { 1431f1b9509cSAlexei Starovoitov switch (func_id) { 1432f1b9509cSAlexei Starovoitov #ifdef CONFIG_NET 1433f1b9509cSAlexei Starovoitov case BPF_FUNC_skb_output: 1434f1b9509cSAlexei Starovoitov return &bpf_skb_output_proto; 1435d831ee84SEelco Chaudron case BPF_FUNC_xdp_output: 1436d831ee84SEelco Chaudron return &bpf_xdp_output_proto; 1437af7ec138SYonghong Song case BPF_FUNC_skc_to_tcp6_sock: 1438af7ec138SYonghong Song return &bpf_skc_to_tcp6_sock_proto; 1439478cfbdfSYonghong Song case BPF_FUNC_skc_to_tcp_sock: 1440478cfbdfSYonghong Song return &bpf_skc_to_tcp_sock_proto; 1441478cfbdfSYonghong Song case BPF_FUNC_skc_to_tcp_timewait_sock: 1442478cfbdfSYonghong Song return &bpf_skc_to_tcp_timewait_sock_proto; 1443478cfbdfSYonghong Song case BPF_FUNC_skc_to_tcp_request_sock: 1444478cfbdfSYonghong Song return &bpf_skc_to_tcp_request_sock_proto; 14450d4fad3eSYonghong Song case BPF_FUNC_skc_to_udp6_sock: 14460d4fad3eSYonghong Song return &bpf_skc_to_udp6_sock_proto; 14478e4597c6SMartin KaFai Lau case BPF_FUNC_sk_storage_get: 14488e4597c6SMartin KaFai Lau return &bpf_sk_storage_get_tracing_proto; 14498e4597c6SMartin KaFai Lau case BPF_FUNC_sk_storage_delete: 14508e4597c6SMartin KaFai Lau return &bpf_sk_storage_delete_tracing_proto; 1451b60da495SFlorent Revest case BPF_FUNC_sock_from_file: 1452b60da495SFlorent Revest return &bpf_sock_from_file_proto; 1453c5dbb89fSFlorent Revest case BPF_FUNC_get_socket_cookie: 1454c5dbb89fSFlorent Revest return &bpf_get_socket_ptr_cookie_proto; 1455f1b9509cSAlexei Starovoitov #endif 1456492e639fSYonghong Song case BPF_FUNC_seq_printf: 1457492e639fSYonghong Song return prog->expected_attach_type == BPF_TRACE_ITER ? 1458492e639fSYonghong Song &bpf_seq_printf_proto : 1459492e639fSYonghong Song NULL; 1460492e639fSYonghong Song case BPF_FUNC_seq_write: 1461492e639fSYonghong Song return prog->expected_attach_type == BPF_TRACE_ITER ? 1462492e639fSYonghong Song &bpf_seq_write_proto : 1463492e639fSYonghong Song NULL; 1464eb411377SAlan Maguire case BPF_FUNC_seq_printf_btf: 1465eb411377SAlan Maguire return prog->expected_attach_type == BPF_TRACE_ITER ? 1466eb411377SAlan Maguire &bpf_seq_printf_btf_proto : 1467eb411377SAlan Maguire NULL; 14686e22ab9dSJiri Olsa case BPF_FUNC_d_path: 14696e22ab9dSJiri Olsa return &bpf_d_path_proto; 1470f1b9509cSAlexei Starovoitov default: 1471f1b9509cSAlexei Starovoitov return raw_tp_prog_func_proto(func_id, prog); 1472f1b9509cSAlexei Starovoitov } 1473f1b9509cSAlexei Starovoitov } 1474f1b9509cSAlexei Starovoitov 1475c4f6699dSAlexei Starovoitov static bool raw_tp_prog_is_valid_access(int off, int size, 1476c4f6699dSAlexei Starovoitov enum bpf_access_type type, 14775e43f899SAndrey Ignatov const struct bpf_prog *prog, 1478c4f6699dSAlexei Starovoitov struct bpf_insn_access_aux *info) 1479c4f6699dSAlexei Starovoitov { 1480f1b9509cSAlexei Starovoitov if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 1481c4f6699dSAlexei Starovoitov return false; 1482c4f6699dSAlexei Starovoitov if (type != BPF_READ) 1483c4f6699dSAlexei Starovoitov return false; 1484c4f6699dSAlexei Starovoitov if (off % size != 0) 1485c4f6699dSAlexei Starovoitov return false; 148638207291SMartin KaFai Lau return true; 1487f1b9509cSAlexei Starovoitov } 1488f1b9509cSAlexei Starovoitov 1489f1b9509cSAlexei Starovoitov static bool tracing_prog_is_valid_access(int off, int size, 1490f1b9509cSAlexei Starovoitov enum bpf_access_type type, 1491f1b9509cSAlexei Starovoitov const struct bpf_prog *prog, 1492f1b9509cSAlexei Starovoitov struct bpf_insn_access_aux *info) 1493f1b9509cSAlexei Starovoitov { 1494f1b9509cSAlexei Starovoitov if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) 1495f1b9509cSAlexei Starovoitov return false; 1496f1b9509cSAlexei Starovoitov if (type != BPF_READ) 1497f1b9509cSAlexei Starovoitov return false; 1498f1b9509cSAlexei Starovoitov if (off % size != 0) 1499f1b9509cSAlexei Starovoitov return false; 15009e15db66SAlexei Starovoitov return btf_ctx_access(off, size, type, prog, info); 1501c4f6699dSAlexei Starovoitov } 1502c4f6699dSAlexei Starovoitov 15033e7c67d9SKP Singh int __weak bpf_prog_test_run_tracing(struct bpf_prog *prog, 15043e7c67d9SKP Singh const union bpf_attr *kattr, 15053e7c67d9SKP Singh union bpf_attr __user *uattr) 15063e7c67d9SKP Singh { 15073e7c67d9SKP Singh return -ENOTSUPP; 15083e7c67d9SKP Singh } 15093e7c67d9SKP Singh 1510c4f6699dSAlexei Starovoitov const struct bpf_verifier_ops raw_tracepoint_verifier_ops = { 1511c4f6699dSAlexei Starovoitov .get_func_proto = raw_tp_prog_func_proto, 1512c4f6699dSAlexei Starovoitov .is_valid_access = raw_tp_prog_is_valid_access, 1513c4f6699dSAlexei Starovoitov }; 1514c4f6699dSAlexei Starovoitov 1515c4f6699dSAlexei Starovoitov const struct bpf_prog_ops raw_tracepoint_prog_ops = { 1516ebfb4d40SYonghong Song #ifdef CONFIG_NET 15171b4d60ecSSong Liu .test_run = bpf_prog_test_run_raw_tp, 1518ebfb4d40SYonghong Song #endif 1519c4f6699dSAlexei Starovoitov }; 1520c4f6699dSAlexei Starovoitov 1521f1b9509cSAlexei Starovoitov const struct bpf_verifier_ops tracing_verifier_ops = { 1522f1b9509cSAlexei Starovoitov .get_func_proto = tracing_prog_func_proto, 1523f1b9509cSAlexei Starovoitov .is_valid_access = tracing_prog_is_valid_access, 1524f1b9509cSAlexei Starovoitov }; 1525f1b9509cSAlexei Starovoitov 1526f1b9509cSAlexei Starovoitov const struct bpf_prog_ops tracing_prog_ops = { 1527da00d2f1SKP Singh .test_run = bpf_prog_test_run_tracing, 1528f1b9509cSAlexei Starovoitov }; 1529f1b9509cSAlexei Starovoitov 15309df1c28bSMatt Mullins static bool raw_tp_writable_prog_is_valid_access(int off, int size, 15319df1c28bSMatt Mullins enum bpf_access_type type, 15329df1c28bSMatt Mullins const struct bpf_prog *prog, 15339df1c28bSMatt Mullins struct bpf_insn_access_aux *info) 15349df1c28bSMatt Mullins { 15359df1c28bSMatt Mullins if (off == 0) { 15369df1c28bSMatt Mullins if (size != sizeof(u64) || type != BPF_READ) 15379df1c28bSMatt Mullins return false; 15389df1c28bSMatt Mullins info->reg_type = PTR_TO_TP_BUFFER; 15399df1c28bSMatt Mullins } 15409df1c28bSMatt Mullins return raw_tp_prog_is_valid_access(off, size, type, prog, info); 15419df1c28bSMatt Mullins } 15429df1c28bSMatt Mullins 15439df1c28bSMatt Mullins const struct bpf_verifier_ops raw_tracepoint_writable_verifier_ops = { 15449df1c28bSMatt Mullins .get_func_proto = raw_tp_prog_func_proto, 15459df1c28bSMatt Mullins .is_valid_access = raw_tp_writable_prog_is_valid_access, 15469df1c28bSMatt Mullins }; 15479df1c28bSMatt Mullins 15489df1c28bSMatt Mullins const struct bpf_prog_ops raw_tracepoint_writable_prog_ops = { 15499df1c28bSMatt Mullins }; 15509df1c28bSMatt Mullins 15510515e599SAlexei Starovoitov static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type, 15525e43f899SAndrey Ignatov const struct bpf_prog *prog, 155323994631SYonghong Song struct bpf_insn_access_aux *info) 15540515e599SAlexei Starovoitov { 155595da0cdbSTeng Qin const int size_u64 = sizeof(u64); 155631fd8581SYonghong Song 15570515e599SAlexei Starovoitov if (off < 0 || off >= sizeof(struct bpf_perf_event_data)) 15580515e599SAlexei Starovoitov return false; 15590515e599SAlexei Starovoitov if (type != BPF_READ) 15600515e599SAlexei Starovoitov return false; 1561bc23105cSDaniel Borkmann if (off % size != 0) { 1562bc23105cSDaniel Borkmann if (sizeof(unsigned long) != 4) 15630515e599SAlexei Starovoitov return false; 1564bc23105cSDaniel Borkmann if (size != 8) 1565bc23105cSDaniel Borkmann return false; 1566bc23105cSDaniel Borkmann if (off % size != 4) 1567bc23105cSDaniel Borkmann return false; 1568bc23105cSDaniel Borkmann } 156931fd8581SYonghong Song 1570f96da094SDaniel Borkmann switch (off) { 1571f96da094SDaniel Borkmann case bpf_ctx_range(struct bpf_perf_event_data, sample_period): 157295da0cdbSTeng Qin bpf_ctx_record_field_size(info, size_u64); 157395da0cdbSTeng Qin if (!bpf_ctx_narrow_access_ok(off, size, size_u64)) 157495da0cdbSTeng Qin return false; 157595da0cdbSTeng Qin break; 157695da0cdbSTeng Qin case bpf_ctx_range(struct bpf_perf_event_data, addr): 157795da0cdbSTeng Qin bpf_ctx_record_field_size(info, size_u64); 157895da0cdbSTeng Qin if (!bpf_ctx_narrow_access_ok(off, size, size_u64)) 157923994631SYonghong Song return false; 1580f96da094SDaniel Borkmann break; 1581f96da094SDaniel Borkmann default: 15820515e599SAlexei Starovoitov if (size != sizeof(long)) 15830515e599SAlexei Starovoitov return false; 15840515e599SAlexei Starovoitov } 1585f96da094SDaniel Borkmann 15860515e599SAlexei Starovoitov return true; 15870515e599SAlexei Starovoitov } 15880515e599SAlexei Starovoitov 15896b8cc1d1SDaniel Borkmann static u32 pe_prog_convert_ctx_access(enum bpf_access_type type, 15906b8cc1d1SDaniel Borkmann const struct bpf_insn *si, 15910515e599SAlexei Starovoitov struct bpf_insn *insn_buf, 1592f96da094SDaniel Borkmann struct bpf_prog *prog, u32 *target_size) 15930515e599SAlexei Starovoitov { 15940515e599SAlexei Starovoitov struct bpf_insn *insn = insn_buf; 15950515e599SAlexei Starovoitov 15966b8cc1d1SDaniel Borkmann switch (si->off) { 15970515e599SAlexei Starovoitov case offsetof(struct bpf_perf_event_data, sample_period): 1598f035a515SDaniel Borkmann *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 15996b8cc1d1SDaniel Borkmann data), si->dst_reg, si->src_reg, 16000515e599SAlexei Starovoitov offsetof(struct bpf_perf_event_data_kern, data)); 16016b8cc1d1SDaniel Borkmann *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, 1602f96da094SDaniel Borkmann bpf_target_off(struct perf_sample_data, period, 8, 1603f96da094SDaniel Borkmann target_size)); 16040515e599SAlexei Starovoitov break; 160595da0cdbSTeng Qin case offsetof(struct bpf_perf_event_data, addr): 160695da0cdbSTeng Qin *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 160795da0cdbSTeng Qin data), si->dst_reg, si->src_reg, 160895da0cdbSTeng Qin offsetof(struct bpf_perf_event_data_kern, data)); 160995da0cdbSTeng Qin *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, 161095da0cdbSTeng Qin bpf_target_off(struct perf_sample_data, addr, 8, 161195da0cdbSTeng Qin target_size)); 161295da0cdbSTeng Qin break; 16130515e599SAlexei Starovoitov default: 1614f035a515SDaniel Borkmann *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 16156b8cc1d1SDaniel Borkmann regs), si->dst_reg, si->src_reg, 16160515e599SAlexei Starovoitov offsetof(struct bpf_perf_event_data_kern, regs)); 16176b8cc1d1SDaniel Borkmann *insn++ = BPF_LDX_MEM(BPF_SIZEOF(long), si->dst_reg, si->dst_reg, 16186b8cc1d1SDaniel Borkmann si->off); 16190515e599SAlexei Starovoitov break; 16200515e599SAlexei Starovoitov } 16210515e599SAlexei Starovoitov 16220515e599SAlexei Starovoitov return insn - insn_buf; 16230515e599SAlexei Starovoitov } 16240515e599SAlexei Starovoitov 16257de16e3aSJakub Kicinski const struct bpf_verifier_ops perf_event_verifier_ops = { 1626f005afedSYonghong Song .get_func_proto = pe_prog_func_proto, 16270515e599SAlexei Starovoitov .is_valid_access = pe_prog_is_valid_access, 16280515e599SAlexei Starovoitov .convert_ctx_access = pe_prog_convert_ctx_access, 16290515e599SAlexei Starovoitov }; 16307de16e3aSJakub Kicinski 16317de16e3aSJakub Kicinski const struct bpf_prog_ops perf_event_prog_ops = { 16327de16e3aSJakub Kicinski }; 1633e87c6bc3SYonghong Song 1634e87c6bc3SYonghong Song static DEFINE_MUTEX(bpf_event_mutex); 1635e87c6bc3SYonghong Song 1636c8c088baSYonghong Song #define BPF_TRACE_MAX_PROGS 64 1637c8c088baSYonghong Song 1638e87c6bc3SYonghong Song int perf_event_attach_bpf_prog(struct perf_event *event, 1639e87c6bc3SYonghong Song struct bpf_prog *prog) 1640e87c6bc3SYonghong Song { 1641e672db03SStanislav Fomichev struct bpf_prog_array *old_array; 1642e87c6bc3SYonghong Song struct bpf_prog_array *new_array; 1643e87c6bc3SYonghong Song int ret = -EEXIST; 1644e87c6bc3SYonghong Song 16459802d865SJosef Bacik /* 1646b4da3340SMasami Hiramatsu * Kprobe override only works if they are on the function entry, 1647b4da3340SMasami Hiramatsu * and only if they are on the opt-in list. 16489802d865SJosef Bacik */ 16499802d865SJosef Bacik if (prog->kprobe_override && 1650b4da3340SMasami Hiramatsu (!trace_kprobe_on_func_entry(event->tp_event) || 16519802d865SJosef Bacik !trace_kprobe_error_injectable(event->tp_event))) 16529802d865SJosef Bacik return -EINVAL; 16539802d865SJosef Bacik 1654e87c6bc3SYonghong Song mutex_lock(&bpf_event_mutex); 1655e87c6bc3SYonghong Song 1656e87c6bc3SYonghong Song if (event->prog) 165707c41a29SYonghong Song goto unlock; 1658e87c6bc3SYonghong Song 1659e672db03SStanislav Fomichev old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); 1660c8c088baSYonghong Song if (old_array && 1661c8c088baSYonghong Song bpf_prog_array_length(old_array) >= BPF_TRACE_MAX_PROGS) { 1662c8c088baSYonghong Song ret = -E2BIG; 1663c8c088baSYonghong Song goto unlock; 1664c8c088baSYonghong Song } 1665c8c088baSYonghong Song 1666e87c6bc3SYonghong Song ret = bpf_prog_array_copy(old_array, NULL, prog, &new_array); 1667e87c6bc3SYonghong Song if (ret < 0) 166807c41a29SYonghong Song goto unlock; 1669e87c6bc3SYonghong Song 1670e87c6bc3SYonghong Song /* set the new array to event->tp_event and set event->prog */ 1671e87c6bc3SYonghong Song event->prog = prog; 1672e87c6bc3SYonghong Song rcu_assign_pointer(event->tp_event->prog_array, new_array); 1673e87c6bc3SYonghong Song bpf_prog_array_free(old_array); 1674e87c6bc3SYonghong Song 167507c41a29SYonghong Song unlock: 1676e87c6bc3SYonghong Song mutex_unlock(&bpf_event_mutex); 1677e87c6bc3SYonghong Song return ret; 1678e87c6bc3SYonghong Song } 1679e87c6bc3SYonghong Song 1680e87c6bc3SYonghong Song void perf_event_detach_bpf_prog(struct perf_event *event) 1681e87c6bc3SYonghong Song { 1682e672db03SStanislav Fomichev struct bpf_prog_array *old_array; 1683e87c6bc3SYonghong Song struct bpf_prog_array *new_array; 1684e87c6bc3SYonghong Song int ret; 1685e87c6bc3SYonghong Song 1686e87c6bc3SYonghong Song mutex_lock(&bpf_event_mutex); 1687e87c6bc3SYonghong Song 1688e87c6bc3SYonghong Song if (!event->prog) 168907c41a29SYonghong Song goto unlock; 1690e87c6bc3SYonghong Song 1691e672db03SStanislav Fomichev old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); 1692e87c6bc3SYonghong Song ret = bpf_prog_array_copy(old_array, event->prog, NULL, &new_array); 1693170a7e3eSSean Young if (ret == -ENOENT) 1694170a7e3eSSean Young goto unlock; 1695e87c6bc3SYonghong Song if (ret < 0) { 1696e87c6bc3SYonghong Song bpf_prog_array_delete_safe(old_array, event->prog); 1697e87c6bc3SYonghong Song } else { 1698e87c6bc3SYonghong Song rcu_assign_pointer(event->tp_event->prog_array, new_array); 1699e87c6bc3SYonghong Song bpf_prog_array_free(old_array); 1700e87c6bc3SYonghong Song } 1701e87c6bc3SYonghong Song 1702e87c6bc3SYonghong Song bpf_prog_put(event->prog); 1703e87c6bc3SYonghong Song event->prog = NULL; 1704e87c6bc3SYonghong Song 170507c41a29SYonghong Song unlock: 1706e87c6bc3SYonghong Song mutex_unlock(&bpf_event_mutex); 1707e87c6bc3SYonghong Song } 1708f371b304SYonghong Song 1709f4e2298eSYonghong Song int perf_event_query_prog_array(struct perf_event *event, void __user *info) 1710f371b304SYonghong Song { 1711f371b304SYonghong Song struct perf_event_query_bpf __user *uquery = info; 1712f371b304SYonghong Song struct perf_event_query_bpf query = {}; 1713e672db03SStanislav Fomichev struct bpf_prog_array *progs; 17143a38bb98SYonghong Song u32 *ids, prog_cnt, ids_len; 1715f371b304SYonghong Song int ret; 1716f371b304SYonghong Song 1717031258daSAlexey Budankov if (!perfmon_capable()) 1718f371b304SYonghong Song return -EPERM; 1719f371b304SYonghong Song if (event->attr.type != PERF_TYPE_TRACEPOINT) 1720f371b304SYonghong Song return -EINVAL; 1721f371b304SYonghong Song if (copy_from_user(&query, uquery, sizeof(query))) 1722f371b304SYonghong Song return -EFAULT; 17233a38bb98SYonghong Song 17243a38bb98SYonghong Song ids_len = query.ids_len; 17253a38bb98SYonghong Song if (ids_len > BPF_TRACE_MAX_PROGS) 17269c481b90SDaniel Borkmann return -E2BIG; 17273a38bb98SYonghong Song ids = kcalloc(ids_len, sizeof(u32), GFP_USER | __GFP_NOWARN); 17283a38bb98SYonghong Song if (!ids) 17293a38bb98SYonghong Song return -ENOMEM; 17303a38bb98SYonghong Song /* 17313a38bb98SYonghong Song * The above kcalloc returns ZERO_SIZE_PTR when ids_len = 0, which 17323a38bb98SYonghong Song * is required when user only wants to check for uquery->prog_cnt. 17333a38bb98SYonghong Song * There is no need to check for it since the case is handled 17343a38bb98SYonghong Song * gracefully in bpf_prog_array_copy_info. 17353a38bb98SYonghong Song */ 1736f371b304SYonghong Song 1737f371b304SYonghong Song mutex_lock(&bpf_event_mutex); 1738e672db03SStanislav Fomichev progs = bpf_event_rcu_dereference(event->tp_event->prog_array); 1739e672db03SStanislav Fomichev ret = bpf_prog_array_copy_info(progs, ids, ids_len, &prog_cnt); 1740f371b304SYonghong Song mutex_unlock(&bpf_event_mutex); 1741f371b304SYonghong Song 17423a38bb98SYonghong Song if (copy_to_user(&uquery->prog_cnt, &prog_cnt, sizeof(prog_cnt)) || 17433a38bb98SYonghong Song copy_to_user(uquery->ids, ids, ids_len * sizeof(u32))) 17443a38bb98SYonghong Song ret = -EFAULT; 17453a38bb98SYonghong Song 17463a38bb98SYonghong Song kfree(ids); 1747f371b304SYonghong Song return ret; 1748f371b304SYonghong Song } 1749c4f6699dSAlexei Starovoitov 1750c4f6699dSAlexei Starovoitov extern struct bpf_raw_event_map __start__bpf_raw_tp[]; 1751c4f6699dSAlexei Starovoitov extern struct bpf_raw_event_map __stop__bpf_raw_tp[]; 1752c4f6699dSAlexei Starovoitov 1753a38d1107SMatt Mullins struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name) 1754c4f6699dSAlexei Starovoitov { 1755c4f6699dSAlexei Starovoitov struct bpf_raw_event_map *btp = __start__bpf_raw_tp; 1756c4f6699dSAlexei Starovoitov 1757c4f6699dSAlexei Starovoitov for (; btp < __stop__bpf_raw_tp; btp++) { 1758c4f6699dSAlexei Starovoitov if (!strcmp(btp->tp->name, name)) 1759c4f6699dSAlexei Starovoitov return btp; 1760c4f6699dSAlexei Starovoitov } 1761a38d1107SMatt Mullins 1762a38d1107SMatt Mullins return bpf_get_raw_tracepoint_module(name); 1763a38d1107SMatt Mullins } 1764a38d1107SMatt Mullins 1765a38d1107SMatt Mullins void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp) 1766a38d1107SMatt Mullins { 176712cc126dSAndrii Nakryiko struct module *mod; 1768a38d1107SMatt Mullins 176912cc126dSAndrii Nakryiko preempt_disable(); 177012cc126dSAndrii Nakryiko mod = __module_address((unsigned long)btp); 1771a38d1107SMatt Mullins module_put(mod); 177212cc126dSAndrii Nakryiko preempt_enable(); 1773c4f6699dSAlexei Starovoitov } 1774c4f6699dSAlexei Starovoitov 1775c4f6699dSAlexei Starovoitov static __always_inline 1776c4f6699dSAlexei Starovoitov void __bpf_trace_run(struct bpf_prog *prog, u64 *args) 1777c4f6699dSAlexei Starovoitov { 1778f03efe49SThomas Gleixner cant_sleep(); 1779c4f6699dSAlexei Starovoitov rcu_read_lock(); 1780c4f6699dSAlexei Starovoitov (void) BPF_PROG_RUN(prog, args); 1781c4f6699dSAlexei Starovoitov rcu_read_unlock(); 1782c4f6699dSAlexei Starovoitov } 1783c4f6699dSAlexei Starovoitov 1784c4f6699dSAlexei Starovoitov #define UNPACK(...) __VA_ARGS__ 1785c4f6699dSAlexei Starovoitov #define REPEAT_1(FN, DL, X, ...) FN(X) 1786c4f6699dSAlexei Starovoitov #define REPEAT_2(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_1(FN, DL, __VA_ARGS__) 1787c4f6699dSAlexei Starovoitov #define REPEAT_3(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_2(FN, DL, __VA_ARGS__) 1788c4f6699dSAlexei Starovoitov #define REPEAT_4(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_3(FN, DL, __VA_ARGS__) 1789c4f6699dSAlexei Starovoitov #define REPEAT_5(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_4(FN, DL, __VA_ARGS__) 1790c4f6699dSAlexei Starovoitov #define REPEAT_6(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_5(FN, DL, __VA_ARGS__) 1791c4f6699dSAlexei Starovoitov #define REPEAT_7(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_6(FN, DL, __VA_ARGS__) 1792c4f6699dSAlexei Starovoitov #define REPEAT_8(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_7(FN, DL, __VA_ARGS__) 1793c4f6699dSAlexei Starovoitov #define REPEAT_9(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_8(FN, DL, __VA_ARGS__) 1794c4f6699dSAlexei Starovoitov #define REPEAT_10(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_9(FN, DL, __VA_ARGS__) 1795c4f6699dSAlexei Starovoitov #define REPEAT_11(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_10(FN, DL, __VA_ARGS__) 1796c4f6699dSAlexei Starovoitov #define REPEAT_12(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_11(FN, DL, __VA_ARGS__) 1797c4f6699dSAlexei Starovoitov #define REPEAT(X, FN, DL, ...) REPEAT_##X(FN, DL, __VA_ARGS__) 1798c4f6699dSAlexei Starovoitov 1799c4f6699dSAlexei Starovoitov #define SARG(X) u64 arg##X 1800c4f6699dSAlexei Starovoitov #define COPY(X) args[X] = arg##X 1801c4f6699dSAlexei Starovoitov 1802c4f6699dSAlexei Starovoitov #define __DL_COM (,) 1803c4f6699dSAlexei Starovoitov #define __DL_SEM (;) 1804c4f6699dSAlexei Starovoitov 1805c4f6699dSAlexei Starovoitov #define __SEQ_0_11 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 1806c4f6699dSAlexei Starovoitov 1807c4f6699dSAlexei Starovoitov #define BPF_TRACE_DEFN_x(x) \ 1808c4f6699dSAlexei Starovoitov void bpf_trace_run##x(struct bpf_prog *prog, \ 1809c4f6699dSAlexei Starovoitov REPEAT(x, SARG, __DL_COM, __SEQ_0_11)) \ 1810c4f6699dSAlexei Starovoitov { \ 1811c4f6699dSAlexei Starovoitov u64 args[x]; \ 1812c4f6699dSAlexei Starovoitov REPEAT(x, COPY, __DL_SEM, __SEQ_0_11); \ 1813c4f6699dSAlexei Starovoitov __bpf_trace_run(prog, args); \ 1814c4f6699dSAlexei Starovoitov } \ 1815c4f6699dSAlexei Starovoitov EXPORT_SYMBOL_GPL(bpf_trace_run##x) 1816c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(1); 1817c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(2); 1818c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(3); 1819c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(4); 1820c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(5); 1821c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(6); 1822c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(7); 1823c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(8); 1824c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(9); 1825c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(10); 1826c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(11); 1827c4f6699dSAlexei Starovoitov BPF_TRACE_DEFN_x(12); 1828c4f6699dSAlexei Starovoitov 1829c4f6699dSAlexei Starovoitov static int __bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog) 1830c4f6699dSAlexei Starovoitov { 1831c4f6699dSAlexei Starovoitov struct tracepoint *tp = btp->tp; 1832c4f6699dSAlexei Starovoitov 1833c4f6699dSAlexei Starovoitov /* 1834c4f6699dSAlexei Starovoitov * check that program doesn't access arguments beyond what's 1835c4f6699dSAlexei Starovoitov * available in this tracepoint 1836c4f6699dSAlexei Starovoitov */ 1837c4f6699dSAlexei Starovoitov if (prog->aux->max_ctx_offset > btp->num_args * sizeof(u64)) 1838c4f6699dSAlexei Starovoitov return -EINVAL; 1839c4f6699dSAlexei Starovoitov 18409df1c28bSMatt Mullins if (prog->aux->max_tp_access > btp->writable_size) 18419df1c28bSMatt Mullins return -EINVAL; 18429df1c28bSMatt Mullins 1843c4f6699dSAlexei Starovoitov return tracepoint_probe_register(tp, (void *)btp->bpf_func, prog); 1844c4f6699dSAlexei Starovoitov } 1845c4f6699dSAlexei Starovoitov 1846c4f6699dSAlexei Starovoitov int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog) 1847c4f6699dSAlexei Starovoitov { 1848e16ec340SAlexei Starovoitov return __bpf_probe_register(btp, prog); 1849c4f6699dSAlexei Starovoitov } 1850c4f6699dSAlexei Starovoitov 1851c4f6699dSAlexei Starovoitov int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog) 1852c4f6699dSAlexei Starovoitov { 1853e16ec340SAlexei Starovoitov return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, prog); 1854c4f6699dSAlexei Starovoitov } 185541bdc4b4SYonghong Song 185641bdc4b4SYonghong Song int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, 185741bdc4b4SYonghong Song u32 *fd_type, const char **buf, 185841bdc4b4SYonghong Song u64 *probe_offset, u64 *probe_addr) 185941bdc4b4SYonghong Song { 186041bdc4b4SYonghong Song bool is_tracepoint, is_syscall_tp; 186141bdc4b4SYonghong Song struct bpf_prog *prog; 186241bdc4b4SYonghong Song int flags, err = 0; 186341bdc4b4SYonghong Song 186441bdc4b4SYonghong Song prog = event->prog; 186541bdc4b4SYonghong Song if (!prog) 186641bdc4b4SYonghong Song return -ENOENT; 186741bdc4b4SYonghong Song 186841bdc4b4SYonghong Song /* not supporting BPF_PROG_TYPE_PERF_EVENT yet */ 186941bdc4b4SYonghong Song if (prog->type == BPF_PROG_TYPE_PERF_EVENT) 187041bdc4b4SYonghong Song return -EOPNOTSUPP; 187141bdc4b4SYonghong Song 187241bdc4b4SYonghong Song *prog_id = prog->aux->id; 187341bdc4b4SYonghong Song flags = event->tp_event->flags; 187441bdc4b4SYonghong Song is_tracepoint = flags & TRACE_EVENT_FL_TRACEPOINT; 187541bdc4b4SYonghong Song is_syscall_tp = is_syscall_trace_event(event->tp_event); 187641bdc4b4SYonghong Song 187741bdc4b4SYonghong Song if (is_tracepoint || is_syscall_tp) { 187841bdc4b4SYonghong Song *buf = is_tracepoint ? event->tp_event->tp->name 187941bdc4b4SYonghong Song : event->tp_event->name; 188041bdc4b4SYonghong Song *fd_type = BPF_FD_TYPE_TRACEPOINT; 188141bdc4b4SYonghong Song *probe_offset = 0x0; 188241bdc4b4SYonghong Song *probe_addr = 0x0; 188341bdc4b4SYonghong Song } else { 188441bdc4b4SYonghong Song /* kprobe/uprobe */ 188541bdc4b4SYonghong Song err = -EOPNOTSUPP; 188641bdc4b4SYonghong Song #ifdef CONFIG_KPROBE_EVENTS 188741bdc4b4SYonghong Song if (flags & TRACE_EVENT_FL_KPROBE) 188841bdc4b4SYonghong Song err = bpf_get_kprobe_info(event, fd_type, buf, 188941bdc4b4SYonghong Song probe_offset, probe_addr, 189041bdc4b4SYonghong Song event->attr.type == PERF_TYPE_TRACEPOINT); 189141bdc4b4SYonghong Song #endif 189241bdc4b4SYonghong Song #ifdef CONFIG_UPROBE_EVENTS 189341bdc4b4SYonghong Song if (flags & TRACE_EVENT_FL_UPROBE) 189441bdc4b4SYonghong Song err = bpf_get_uprobe_info(event, fd_type, buf, 189541bdc4b4SYonghong Song probe_offset, 189641bdc4b4SYonghong Song event->attr.type == PERF_TYPE_TRACEPOINT); 189741bdc4b4SYonghong Song #endif 189841bdc4b4SYonghong Song } 189941bdc4b4SYonghong Song 190041bdc4b4SYonghong Song return err; 190141bdc4b4SYonghong Song } 1902a38d1107SMatt Mullins 19039db1ff0aSYonghong Song static int __init send_signal_irq_work_init(void) 19049db1ff0aSYonghong Song { 19059db1ff0aSYonghong Song int cpu; 19069db1ff0aSYonghong Song struct send_signal_irq_work *work; 19079db1ff0aSYonghong Song 19089db1ff0aSYonghong Song for_each_possible_cpu(cpu) { 19099db1ff0aSYonghong Song work = per_cpu_ptr(&send_signal_work, cpu); 19109db1ff0aSYonghong Song init_irq_work(&work->irq_work, do_bpf_send_signal); 19119db1ff0aSYonghong Song } 19129db1ff0aSYonghong Song return 0; 19139db1ff0aSYonghong Song } 19149db1ff0aSYonghong Song 19159db1ff0aSYonghong Song subsys_initcall(send_signal_irq_work_init); 19169db1ff0aSYonghong Song 1917a38d1107SMatt Mullins #ifdef CONFIG_MODULES 1918390e99cfSStanislav Fomichev static int bpf_event_notify(struct notifier_block *nb, unsigned long op, 1919390e99cfSStanislav Fomichev void *module) 1920a38d1107SMatt Mullins { 1921a38d1107SMatt Mullins struct bpf_trace_module *btm, *tmp; 1922a38d1107SMatt Mullins struct module *mod = module; 19230340a6b7SPeter Zijlstra int ret = 0; 1924a38d1107SMatt Mullins 1925a38d1107SMatt Mullins if (mod->num_bpf_raw_events == 0 || 1926a38d1107SMatt Mullins (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING)) 19270340a6b7SPeter Zijlstra goto out; 1928a38d1107SMatt Mullins 1929a38d1107SMatt Mullins mutex_lock(&bpf_module_mutex); 1930a38d1107SMatt Mullins 1931a38d1107SMatt Mullins switch (op) { 1932a38d1107SMatt Mullins case MODULE_STATE_COMING: 1933a38d1107SMatt Mullins btm = kzalloc(sizeof(*btm), GFP_KERNEL); 1934a38d1107SMatt Mullins if (btm) { 1935a38d1107SMatt Mullins btm->module = module; 1936a38d1107SMatt Mullins list_add(&btm->list, &bpf_trace_modules); 19370340a6b7SPeter Zijlstra } else { 19380340a6b7SPeter Zijlstra ret = -ENOMEM; 1939a38d1107SMatt Mullins } 1940a38d1107SMatt Mullins break; 1941a38d1107SMatt Mullins case MODULE_STATE_GOING: 1942a38d1107SMatt Mullins list_for_each_entry_safe(btm, tmp, &bpf_trace_modules, list) { 1943a38d1107SMatt Mullins if (btm->module == module) { 1944a38d1107SMatt Mullins list_del(&btm->list); 1945a38d1107SMatt Mullins kfree(btm); 1946a38d1107SMatt Mullins break; 1947a38d1107SMatt Mullins } 1948a38d1107SMatt Mullins } 1949a38d1107SMatt Mullins break; 1950a38d1107SMatt Mullins } 1951a38d1107SMatt Mullins 1952a38d1107SMatt Mullins mutex_unlock(&bpf_module_mutex); 1953a38d1107SMatt Mullins 19540340a6b7SPeter Zijlstra out: 19550340a6b7SPeter Zijlstra return notifier_from_errno(ret); 1956a38d1107SMatt Mullins } 1957a38d1107SMatt Mullins 1958a38d1107SMatt Mullins static struct notifier_block bpf_module_nb = { 1959a38d1107SMatt Mullins .notifier_call = bpf_event_notify, 1960a38d1107SMatt Mullins }; 1961a38d1107SMatt Mullins 1962390e99cfSStanislav Fomichev static int __init bpf_event_init(void) 1963a38d1107SMatt Mullins { 1964a38d1107SMatt Mullins register_module_notifier(&bpf_module_nb); 1965a38d1107SMatt Mullins return 0; 1966a38d1107SMatt Mullins } 1967a38d1107SMatt Mullins 1968a38d1107SMatt Mullins fs_initcall(bpf_event_init); 1969a38d1107SMatt Mullins #endif /* CONFIG_MODULES */ 1970