17f5a08c7SBeau Belgrave // SPDX-License-Identifier: GPL-2.0-only
27f5a08c7SBeau Belgrave /*
37f5a08c7SBeau Belgrave * Copyright (c) 2021, Microsoft Corporation.
47f5a08c7SBeau Belgrave *
57f5a08c7SBeau Belgrave * Authors:
67f5a08c7SBeau Belgrave * Beau Belgrave <beaub@linux.microsoft.com>
77f5a08c7SBeau Belgrave */
87f5a08c7SBeau Belgrave
97f5a08c7SBeau Belgrave #include <linux/bitmap.h>
107f5a08c7SBeau Belgrave #include <linux/cdev.h>
117f5a08c7SBeau Belgrave #include <linux/hashtable.h>
127f5a08c7SBeau Belgrave #include <linux/list.h>
137f5a08c7SBeau Belgrave #include <linux/io.h>
147f5a08c7SBeau Belgrave #include <linux/uio.h>
157f5a08c7SBeau Belgrave #include <linux/ioctl.h>
167f5a08c7SBeau Belgrave #include <linux/jhash.h>
17d401b724SBeau Belgrave #include <linux/refcount.h>
187f5a08c7SBeau Belgrave #include <linux/trace_events.h>
197f5a08c7SBeau Belgrave #include <linux/tracefs.h>
207f5a08c7SBeau Belgrave #include <linux/types.h>
217f5a08c7SBeau Belgrave #include <linux/uaccess.h>
2272357590SBeau Belgrave #include <linux/highmem.h>
23ce58e96eSBeau Belgrave #include <linux/init.h>
245cfff569SSteven Rostedt (Google) #include <linux/user_events.h>
257f5a08c7SBeau Belgrave #include "trace_dynevent.h"
264bec284cSSteven Rostedt (Google) #include "trace_output.h"
274bec284cSSteven Rostedt (Google) #include "trace.h"
287f5a08c7SBeau Belgrave
297f5a08c7SBeau Belgrave #define USER_EVENTS_PREFIX_LEN (sizeof(USER_EVENTS_PREFIX)-1)
307f5a08c7SBeau Belgrave
317f5a08c7SBeau Belgrave #define FIELD_DEPTH_TYPE 0
327f5a08c7SBeau Belgrave #define FIELD_DEPTH_NAME 1
337f5a08c7SBeau Belgrave #define FIELD_DEPTH_SIZE 2
347f5a08c7SBeau Belgrave
357f5a08c7SBeau Belgrave /* Limit how long of an event name plus args within the subsystem. */
367f5a08c7SBeau Belgrave #define MAX_EVENT_DESC 512
377f5a08c7SBeau Belgrave #define EVENT_NAME(user_event) ((user_event)->tracepoint.name)
387f5a08c7SBeau Belgrave #define MAX_FIELD_ARRAY_SIZE 1024
397f5a08c7SBeau Belgrave
4039d6d08bSBeau Belgrave /*
4139d6d08bSBeau Belgrave * Internal bits (kernel side only) to keep track of connected probes:
4239d6d08bSBeau Belgrave * These are used when status is requested in text form about an event. These
4339d6d08bSBeau Belgrave * bits are compared against an internal byte on the event to determine which
4439d6d08bSBeau Belgrave * probes to print out to the user.
4539d6d08bSBeau Belgrave *
4639d6d08bSBeau Belgrave * These do not reflect the mapped bytes between the user and kernel space.
4739d6d08bSBeau Belgrave */
4839d6d08bSBeau Belgrave #define EVENT_STATUS_FTRACE BIT(0)
4939d6d08bSBeau Belgrave #define EVENT_STATUS_PERF BIT(1)
5039d6d08bSBeau Belgrave #define EVENT_STATUS_OTHER BIT(7)
5139d6d08bSBeau Belgrave
52e5d27181SBeau Belgrave /*
53a65442edSBeau Belgrave * User register flags are not allowed yet, keep them here until we are
54a65442edSBeau Belgrave * ready to expose them out to the user ABI.
55a65442edSBeau Belgrave */
56a65442edSBeau Belgrave enum user_reg_flag {
57a65442edSBeau Belgrave /* Event will not delete upon last reference closing */
58a65442edSBeau Belgrave USER_EVENT_REG_PERSIST = 1U << 0,
59a65442edSBeau Belgrave
60a65442edSBeau Belgrave /* This value or above is currently non-ABI */
61a65442edSBeau Belgrave USER_EVENT_REG_MAX = 1U << 1,
62a65442edSBeau Belgrave };
63a65442edSBeau Belgrave
64a65442edSBeau Belgrave /*
6572357590SBeau Belgrave * Stores the system name, tables, and locks for a group of events. This
6672357590SBeau Belgrave * allows isolation for events by various means.
67e5d27181SBeau Belgrave */
68e5d27181SBeau Belgrave struct user_event_group {
69e5d27181SBeau Belgrave char *system_name;
70e5d27181SBeau Belgrave struct hlist_node node;
71e5d27181SBeau Belgrave struct mutex reg_mutex;
72e5d27181SBeau Belgrave DECLARE_HASHTABLE(register_table, 8);
73e5d27181SBeau Belgrave };
747f5a08c7SBeau Belgrave
75e5d27181SBeau Belgrave /* Group for init_user_ns mapping, top-most group */
76e5d27181SBeau Belgrave static struct user_event_group *init_group;
777f5a08c7SBeau Belgrave
78ce58e96eSBeau Belgrave /* Max allowed events for the whole system */
79ce58e96eSBeau Belgrave static unsigned int max_user_events = 32768;
80ce58e96eSBeau Belgrave
81ce58e96eSBeau Belgrave /* Current number of events on the whole system */
82ce58e96eSBeau Belgrave static unsigned int current_user_events;
83ce58e96eSBeau Belgrave
847f5a08c7SBeau Belgrave /*
857f5a08c7SBeau Belgrave * Stores per-event properties, as users register events
867f5a08c7SBeau Belgrave * within a file a user_event might be created if it does not
877f5a08c7SBeau Belgrave * already exist. These are globally used and their lifetime
887f5a08c7SBeau Belgrave * is tied to the refcnt member. These cannot go away until the
89d401b724SBeau Belgrave * refcnt reaches one.
907f5a08c7SBeau Belgrave */
917f5a08c7SBeau Belgrave struct user_event {
92e5d27181SBeau Belgrave struct user_event_group *group;
937f5a08c7SBeau Belgrave struct tracepoint tracepoint;
947f5a08c7SBeau Belgrave struct trace_event_call call;
957f5a08c7SBeau Belgrave struct trace_event_class class;
967f5a08c7SBeau Belgrave struct dyn_event devent;
977f5a08c7SBeau Belgrave struct hlist_node node;
987f5a08c7SBeau Belgrave struct list_head fields;
992467cda1SBeau Belgrave struct list_head validators;
100a65442edSBeau Belgrave struct work_struct put_work;
101d401b724SBeau Belgrave refcount_t refcnt;
1022467cda1SBeau Belgrave int min_size;
103b08d7258SBeau Belgrave int reg_flags;
10439d6d08bSBeau Belgrave char status;
1057f5a08c7SBeau Belgrave };
1067f5a08c7SBeau Belgrave
1077f5a08c7SBeau Belgrave /*
10872357590SBeau Belgrave * Stores per-mm/event properties that enable an address to be
10972357590SBeau Belgrave * updated properly for each task. As tasks are forked, we use
11072357590SBeau Belgrave * these to track enablement sites that are tied to an event.
11172357590SBeau Belgrave */
11272357590SBeau Belgrave struct user_event_enabler {
113dcbd1ac2SBeau Belgrave struct list_head mm_enablers_link;
11472357590SBeau Belgrave struct user_event *event;
11572357590SBeau Belgrave unsigned long addr;
11672357590SBeau Belgrave
11772357590SBeau Belgrave /* Track enable bit, flags, etc. Aligned for bitops. */
118ee7751b5SBeau Belgrave unsigned long values;
11972357590SBeau Belgrave };
12072357590SBeau Belgrave
12172357590SBeau Belgrave /* Bits 0-5 are for the bit to update upon enable/disable (0-63 allowed) */
12272357590SBeau Belgrave #define ENABLE_VAL_BIT_MASK 0x3F
12372357590SBeau Belgrave
12481f8fb65SBeau Belgrave /* Bit 6 is for faulting status of enablement */
12581f8fb65SBeau Belgrave #define ENABLE_VAL_FAULTING_BIT 6
12681f8fb65SBeau Belgrave
127dcb8177cSBeau Belgrave /* Bit 7 is for freeing status of enablement */
128dcb8177cSBeau Belgrave #define ENABLE_VAL_FREEING_BIT 7
129dcb8177cSBeau Belgrave
130*2de9ee94SBeau Belgrave /* Bit 8 is for marking 32-bit on 64-bit */
131*2de9ee94SBeau Belgrave #define ENABLE_VAL_32_ON_64_BIT 8
132*2de9ee94SBeau Belgrave
133*2de9ee94SBeau Belgrave #define ENABLE_VAL_COMPAT_MASK (1 << ENABLE_VAL_32_ON_64_BIT)
134*2de9ee94SBeau Belgrave
135*2de9ee94SBeau Belgrave /* Only duplicate the bit and compat values */
136*2de9ee94SBeau Belgrave #define ENABLE_VAL_DUP_MASK (ENABLE_VAL_BIT_MASK | ENABLE_VAL_COMPAT_MASK)
13772357590SBeau Belgrave
138ee7751b5SBeau Belgrave #define ENABLE_BITOPS(e) (&(e)->values)
139ee7751b5SBeau Belgrave
140ee7751b5SBeau Belgrave #define ENABLE_BIT(e) ((int)((e)->values & ENABLE_VAL_BIT_MASK))
14181f8fb65SBeau Belgrave
14281f8fb65SBeau Belgrave /* Used for asynchronous faulting in of pages */
14381f8fb65SBeau Belgrave struct user_event_enabler_fault {
14481f8fb65SBeau Belgrave struct work_struct work;
14581f8fb65SBeau Belgrave struct user_event_mm *mm;
14681f8fb65SBeau Belgrave struct user_event_enabler *enabler;
14741d8fba1SBeau Belgrave int attempt;
14881f8fb65SBeau Belgrave };
14981f8fb65SBeau Belgrave
15081f8fb65SBeau Belgrave static struct kmem_cache *fault_cache;
15181f8fb65SBeau Belgrave
15272357590SBeau Belgrave /* Global list of memory descriptors using user_events */
15372357590SBeau Belgrave static LIST_HEAD(user_event_mms);
15472357590SBeau Belgrave static DEFINE_SPINLOCK(user_event_mms_lock);
15572357590SBeau Belgrave
15672357590SBeau Belgrave /*
1577f5a08c7SBeau Belgrave * Stores per-file events references, as users register events
1587f5a08c7SBeau Belgrave * within a file this structure is modified and freed via RCU.
1597f5a08c7SBeau Belgrave * The lifetime of this struct is tied to the lifetime of the file.
1607f5a08c7SBeau Belgrave * These are not shared and only accessible by the file that created it.
1617f5a08c7SBeau Belgrave */
1627f5a08c7SBeau Belgrave struct user_event_refs {
1637f5a08c7SBeau Belgrave struct rcu_head rcu;
1647f5a08c7SBeau Belgrave int count;
1657f5a08c7SBeau Belgrave struct user_event *events[];
1667f5a08c7SBeau Belgrave };
1677f5a08c7SBeau Belgrave
168e5d27181SBeau Belgrave struct user_event_file_info {
169e5d27181SBeau Belgrave struct user_event_group *group;
170e5d27181SBeau Belgrave struct user_event_refs *refs;
171e5d27181SBeau Belgrave };
172e5d27181SBeau Belgrave
1732467cda1SBeau Belgrave #define VALIDATOR_ENSURE_NULL (1 << 0)
1742467cda1SBeau Belgrave #define VALIDATOR_REL (1 << 1)
1752467cda1SBeau Belgrave
1762467cda1SBeau Belgrave struct user_event_validator {
177dcbd1ac2SBeau Belgrave struct list_head user_event_link;
1782467cda1SBeau Belgrave int offset;
1792467cda1SBeau Belgrave int flags;
1802467cda1SBeau Belgrave };
1812467cda1SBeau Belgrave
align_addr_bit(unsigned long * addr,int * bit,unsigned long * flags)182*2de9ee94SBeau Belgrave static inline void align_addr_bit(unsigned long *addr, int *bit,
183*2de9ee94SBeau Belgrave unsigned long *flags)
184*2de9ee94SBeau Belgrave {
185*2de9ee94SBeau Belgrave if (IS_ALIGNED(*addr, sizeof(long))) {
186*2de9ee94SBeau Belgrave #ifdef __BIG_ENDIAN
187*2de9ee94SBeau Belgrave /* 32 bit on BE 64 bit requires a 32 bit offset when aligned. */
188*2de9ee94SBeau Belgrave if (test_bit(ENABLE_VAL_32_ON_64_BIT, flags))
189*2de9ee94SBeau Belgrave *bit += 32;
190*2de9ee94SBeau Belgrave #endif
191*2de9ee94SBeau Belgrave return;
192*2de9ee94SBeau Belgrave }
193*2de9ee94SBeau Belgrave
194*2de9ee94SBeau Belgrave *addr = ALIGN_DOWN(*addr, sizeof(long));
195*2de9ee94SBeau Belgrave
196*2de9ee94SBeau Belgrave /*
197*2de9ee94SBeau Belgrave * We only support 32 and 64 bit values. The only time we need
198*2de9ee94SBeau Belgrave * to align is a 32 bit value on a 64 bit kernel, which on LE
199*2de9ee94SBeau Belgrave * is always 32 bits, and on BE requires no change when unaligned.
200*2de9ee94SBeau Belgrave */
201*2de9ee94SBeau Belgrave #ifdef __LITTLE_ENDIAN
202*2de9ee94SBeau Belgrave *bit += 32;
203*2de9ee94SBeau Belgrave #endif
204*2de9ee94SBeau Belgrave }
205*2de9ee94SBeau Belgrave
2060279400aSBeau Belgrave typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i,
2072467cda1SBeau Belgrave void *tpdata, bool *faulted);
2087f5a08c7SBeau Belgrave
209e5d27181SBeau Belgrave static int user_event_parse(struct user_event_group *group, char *name,
210e5d27181SBeau Belgrave char *args, char *flags,
211b08d7258SBeau Belgrave struct user_event **newuser, int reg_flags);
2127f5a08c7SBeau Belgrave
21372357590SBeau Belgrave static struct user_event_mm *user_event_mm_get(struct user_event_mm *mm);
21472357590SBeau Belgrave static struct user_event_mm *user_event_mm_get_all(struct user_event *user);
21572357590SBeau Belgrave static void user_event_mm_put(struct user_event_mm *mm);
216a65442edSBeau Belgrave static int destroy_user_event(struct user_event *user);
21772357590SBeau Belgrave
user_event_key(char * name)2187f5a08c7SBeau Belgrave static u32 user_event_key(char *name)
2197f5a08c7SBeau Belgrave {
2207f5a08c7SBeau Belgrave return jhash(name, strlen(name), 0);
2217f5a08c7SBeau Belgrave }
2227f5a08c7SBeau Belgrave
user_event_get(struct user_event * user)223f0dbf6fdSBeau Belgrave static struct user_event *user_event_get(struct user_event *user)
224f0dbf6fdSBeau Belgrave {
225f0dbf6fdSBeau Belgrave refcount_inc(&user->refcnt);
226f0dbf6fdSBeau Belgrave
227f0dbf6fdSBeau Belgrave return user;
228f0dbf6fdSBeau Belgrave }
229f0dbf6fdSBeau Belgrave
delayed_destroy_user_event(struct work_struct * work)230a65442edSBeau Belgrave static void delayed_destroy_user_event(struct work_struct *work)
231a65442edSBeau Belgrave {
232a65442edSBeau Belgrave struct user_event *user = container_of(
233a65442edSBeau Belgrave work, struct user_event, put_work);
234a65442edSBeau Belgrave
235a65442edSBeau Belgrave mutex_lock(&event_mutex);
236a65442edSBeau Belgrave
237a65442edSBeau Belgrave if (!refcount_dec_and_test(&user->refcnt))
238a65442edSBeau Belgrave goto out;
239a65442edSBeau Belgrave
240a65442edSBeau Belgrave if (destroy_user_event(user)) {
241a65442edSBeau Belgrave /*
242a65442edSBeau Belgrave * The only reason this would fail here is if we cannot
243a65442edSBeau Belgrave * update the visibility of the event. In this case the
244a65442edSBeau Belgrave * event stays in the hashtable, waiting for someone to
245a65442edSBeau Belgrave * attempt to delete it later.
246a65442edSBeau Belgrave */
247a65442edSBeau Belgrave pr_warn("user_events: Unable to delete event\n");
248a65442edSBeau Belgrave refcount_set(&user->refcnt, 1);
249a65442edSBeau Belgrave }
250a65442edSBeau Belgrave out:
251a65442edSBeau Belgrave mutex_unlock(&event_mutex);
252a65442edSBeau Belgrave }
253a65442edSBeau Belgrave
user_event_put(struct user_event * user,bool locked)254f0dbf6fdSBeau Belgrave static void user_event_put(struct user_event *user, bool locked)
255f0dbf6fdSBeau Belgrave {
256a65442edSBeau Belgrave bool delete;
257f0dbf6fdSBeau Belgrave
258f0dbf6fdSBeau Belgrave if (unlikely(!user))
259f0dbf6fdSBeau Belgrave return;
260f0dbf6fdSBeau Belgrave
261a65442edSBeau Belgrave /*
262a65442edSBeau Belgrave * When the event is not enabled for auto-delete there will always
263a65442edSBeau Belgrave * be at least 1 reference to the event. During the event creation
264a65442edSBeau Belgrave * we initially set the refcnt to 2 to achieve this. In those cases
265a65442edSBeau Belgrave * the caller must acquire event_mutex and after decrement check if
266a65442edSBeau Belgrave * the refcnt is 1, meaning this is the last reference. When auto
267a65442edSBeau Belgrave * delete is enabled, there will only be 1 ref, IE: refcnt will be
268a65442edSBeau Belgrave * only set to 1 during creation to allow the below checks to go
269a65442edSBeau Belgrave * through upon the last put. The last put must always be done with
270a65442edSBeau Belgrave * the event mutex held.
271a65442edSBeau Belgrave */
272a65442edSBeau Belgrave if (!locked) {
273a65442edSBeau Belgrave lockdep_assert_not_held(&event_mutex);
274a65442edSBeau Belgrave delete = refcount_dec_and_mutex_lock(&user->refcnt, &event_mutex);
275a65442edSBeau Belgrave } else {
276a65442edSBeau Belgrave lockdep_assert_held(&event_mutex);
277a65442edSBeau Belgrave delete = refcount_dec_and_test(&user->refcnt);
278a65442edSBeau Belgrave }
279a65442edSBeau Belgrave
280a65442edSBeau Belgrave if (!delete)
281a65442edSBeau Belgrave return;
282a65442edSBeau Belgrave
283a65442edSBeau Belgrave /*
284a65442edSBeau Belgrave * We now have the event_mutex in all cases, which ensures that
285a65442edSBeau Belgrave * no new references will be taken until event_mutex is released.
286a65442edSBeau Belgrave * New references come through find_user_event(), which requires
287a65442edSBeau Belgrave * the event_mutex to be held.
288a65442edSBeau Belgrave */
289a65442edSBeau Belgrave
290a65442edSBeau Belgrave if (user->reg_flags & USER_EVENT_REG_PERSIST) {
291a65442edSBeau Belgrave /* We should not get here when persist flag is set */
292a65442edSBeau Belgrave pr_alert("BUG: Auto-delete engaged on persistent event\n");
293a65442edSBeau Belgrave goto out;
294a65442edSBeau Belgrave }
295a65442edSBeau Belgrave
296a65442edSBeau Belgrave /*
297a65442edSBeau Belgrave * Unfortunately we have to attempt the actual destroy in a work
298a65442edSBeau Belgrave * queue. This is because not all cases handle a trace_event_call
299a65442edSBeau Belgrave * being removed within the class->reg() operation for unregister.
300a65442edSBeau Belgrave */
301a65442edSBeau Belgrave INIT_WORK(&user->put_work, delayed_destroy_user_event);
302a65442edSBeau Belgrave
303a65442edSBeau Belgrave /*
304a65442edSBeau Belgrave * Since the event is still in the hashtable, we have to re-inc
305a65442edSBeau Belgrave * the ref count to 1. This count will be decremented and checked
306a65442edSBeau Belgrave * in the work queue to ensure it's still the last ref. This is
307a65442edSBeau Belgrave * needed because a user-process could register the same event in
308a65442edSBeau Belgrave * between the time of event_mutex release and the work queue
309a65442edSBeau Belgrave * running the delayed destroy. If we removed the item now from
310a65442edSBeau Belgrave * the hashtable, this would result in a timing window where a
311a65442edSBeau Belgrave * user process would fail a register because the trace_event_call
312a65442edSBeau Belgrave * register would fail in the tracing layers.
313a65442edSBeau Belgrave */
314a65442edSBeau Belgrave refcount_set(&user->refcnt, 1);
315a65442edSBeau Belgrave
316a65442edSBeau Belgrave if (WARN_ON_ONCE(!schedule_work(&user->put_work))) {
317a65442edSBeau Belgrave /*
318a65442edSBeau Belgrave * If we fail we must wait for an admin to attempt delete or
319a65442edSBeau Belgrave * another register/close of the event, whichever is first.
320a65442edSBeau Belgrave */
321a65442edSBeau Belgrave pr_warn("user_events: Unable to queue delayed destroy\n");
322a65442edSBeau Belgrave }
323a65442edSBeau Belgrave out:
324a65442edSBeau Belgrave /* Ensure if we didn't have event_mutex before we unlock it */
325a65442edSBeau Belgrave if (!locked)
326a65442edSBeau Belgrave mutex_unlock(&event_mutex);
327f0dbf6fdSBeau Belgrave }
328f0dbf6fdSBeau Belgrave
user_event_group_destroy(struct user_event_group * group)329e5d27181SBeau Belgrave static void user_event_group_destroy(struct user_event_group *group)
330e5d27181SBeau Belgrave {
331e5d27181SBeau Belgrave kfree(group->system_name);
332e5d27181SBeau Belgrave kfree(group);
333e5d27181SBeau Belgrave }
334e5d27181SBeau Belgrave
user_event_group_system_name(void)335ed0e0ae0SBeau Belgrave static char *user_event_group_system_name(void)
336e5d27181SBeau Belgrave {
337e5d27181SBeau Belgrave char *system_name;
338e5d27181SBeau Belgrave int len = sizeof(USER_EVENTS_SYSTEM) + 1;
339e5d27181SBeau Belgrave
340e5d27181SBeau Belgrave system_name = kmalloc(len, GFP_KERNEL);
341e5d27181SBeau Belgrave
342e5d27181SBeau Belgrave if (!system_name)
343e5d27181SBeau Belgrave return NULL;
344e5d27181SBeau Belgrave
345e5d27181SBeau Belgrave snprintf(system_name, len, "%s", USER_EVENTS_SYSTEM);
346e5d27181SBeau Belgrave
347e5d27181SBeau Belgrave return system_name;
348e5d27181SBeau Belgrave }
349e5d27181SBeau Belgrave
current_user_event_group(void)350e5d27181SBeau Belgrave static struct user_event_group *current_user_event_group(void)
351e5d27181SBeau Belgrave {
352ed0e0ae0SBeau Belgrave return init_group;
353e5d27181SBeau Belgrave }
354e5d27181SBeau Belgrave
user_event_group_create(void)355ed0e0ae0SBeau Belgrave static struct user_event_group *user_event_group_create(void)
356e5d27181SBeau Belgrave {
357e5d27181SBeau Belgrave struct user_event_group *group;
358e5d27181SBeau Belgrave
359e5d27181SBeau Belgrave group = kzalloc(sizeof(*group), GFP_KERNEL);
360e5d27181SBeau Belgrave
361e5d27181SBeau Belgrave if (!group)
362e5d27181SBeau Belgrave return NULL;
363e5d27181SBeau Belgrave
364ed0e0ae0SBeau Belgrave group->system_name = user_event_group_system_name();
365e5d27181SBeau Belgrave
366e5d27181SBeau Belgrave if (!group->system_name)
367e5d27181SBeau Belgrave goto error;
368e5d27181SBeau Belgrave
369e5d27181SBeau Belgrave mutex_init(&group->reg_mutex);
370e5d27181SBeau Belgrave hash_init(group->register_table);
371e5d27181SBeau Belgrave
372e5d27181SBeau Belgrave return group;
373e5d27181SBeau Belgrave error:
374e5d27181SBeau Belgrave if (group)
375e5d27181SBeau Belgrave user_event_group_destroy(group);
376e5d27181SBeau Belgrave
377e5d27181SBeau Belgrave return NULL;
378e5d27181SBeau Belgrave };
379e5d27181SBeau Belgrave
user_event_enabler_destroy(struct user_event_enabler * enabler,bool locked)380f0dbf6fdSBeau Belgrave static void user_event_enabler_destroy(struct user_event_enabler *enabler,
381f0dbf6fdSBeau Belgrave bool locked)
38239d6d08bSBeau Belgrave {
383dcbd1ac2SBeau Belgrave list_del_rcu(&enabler->mm_enablers_link);
38439d6d08bSBeau Belgrave
38572357590SBeau Belgrave /* No longer tracking the event via the enabler */
386f0dbf6fdSBeau Belgrave user_event_put(enabler->event, locked);
38772357590SBeau Belgrave
38872357590SBeau Belgrave kfree(enabler);
38939d6d08bSBeau Belgrave }
39039d6d08bSBeau Belgrave
user_event_mm_fault_in(struct user_event_mm * mm,unsigned long uaddr,int attempt)39141d8fba1SBeau Belgrave static int user_event_mm_fault_in(struct user_event_mm *mm, unsigned long uaddr,
39241d8fba1SBeau Belgrave int attempt)
39339d6d08bSBeau Belgrave {
39472357590SBeau Belgrave bool unlocked;
39572357590SBeau Belgrave int ret;
39639d6d08bSBeau Belgrave
39741d8fba1SBeau Belgrave /*
39841d8fba1SBeau Belgrave * Normally this is low, ensure that it cannot be taken advantage of by
39941d8fba1SBeau Belgrave * bad user processes to cause excessive looping.
40041d8fba1SBeau Belgrave */
40141d8fba1SBeau Belgrave if (attempt > 10)
40241d8fba1SBeau Belgrave return -EFAULT;
40341d8fba1SBeau Belgrave
40472357590SBeau Belgrave mmap_read_lock(mm->mm);
40572357590SBeau Belgrave
40672357590SBeau Belgrave /* Ensure MM has tasks, cannot use after exit_mm() */
40772357590SBeau Belgrave if (refcount_read(&mm->tasks) == 0) {
40872357590SBeau Belgrave ret = -ENOENT;
40972357590SBeau Belgrave goto out;
41072357590SBeau Belgrave }
41172357590SBeau Belgrave
41272357590SBeau Belgrave ret = fixup_user_fault(mm->mm, uaddr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE,
41372357590SBeau Belgrave &unlocked);
41472357590SBeau Belgrave out:
41572357590SBeau Belgrave mmap_read_unlock(mm->mm);
41672357590SBeau Belgrave
41772357590SBeau Belgrave return ret;
41872357590SBeau Belgrave }
41972357590SBeau Belgrave
42072357590SBeau Belgrave static int user_event_enabler_write(struct user_event_mm *mm,
42181f8fb65SBeau Belgrave struct user_event_enabler *enabler,
42241d8fba1SBeau Belgrave bool fixup_fault, int *attempt);
42381f8fb65SBeau Belgrave
user_event_enabler_fault_fixup(struct work_struct * work)42481f8fb65SBeau Belgrave static void user_event_enabler_fault_fixup(struct work_struct *work)
42581f8fb65SBeau Belgrave {
42681f8fb65SBeau Belgrave struct user_event_enabler_fault *fault = container_of(
42781f8fb65SBeau Belgrave work, struct user_event_enabler_fault, work);
42881f8fb65SBeau Belgrave struct user_event_enabler *enabler = fault->enabler;
42981f8fb65SBeau Belgrave struct user_event_mm *mm = fault->mm;
43081f8fb65SBeau Belgrave unsigned long uaddr = enabler->addr;
43141d8fba1SBeau Belgrave int attempt = fault->attempt;
43281f8fb65SBeau Belgrave int ret;
43381f8fb65SBeau Belgrave
43441d8fba1SBeau Belgrave ret = user_event_mm_fault_in(mm, uaddr, attempt);
43581f8fb65SBeau Belgrave
43681f8fb65SBeau Belgrave if (ret && ret != -ENOENT) {
43781f8fb65SBeau Belgrave struct user_event *user = enabler->event;
43881f8fb65SBeau Belgrave
43981f8fb65SBeau Belgrave pr_warn("user_events: Fault for mm: 0x%pK @ 0x%llx event: %s\n",
44081f8fb65SBeau Belgrave mm->mm, (unsigned long long)uaddr, EVENT_NAME(user));
44181f8fb65SBeau Belgrave }
44281f8fb65SBeau Belgrave
44381f8fb65SBeau Belgrave /* Prevent state changes from racing */
44481f8fb65SBeau Belgrave mutex_lock(&event_mutex);
44581f8fb65SBeau Belgrave
446dcb8177cSBeau Belgrave /* User asked for enabler to be removed during fault */
447dcb8177cSBeau Belgrave if (test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler))) {
448f0dbf6fdSBeau Belgrave user_event_enabler_destroy(enabler, true);
449dcb8177cSBeau Belgrave goto out;
450dcb8177cSBeau Belgrave }
451dcb8177cSBeau Belgrave
45281f8fb65SBeau Belgrave /*
45381f8fb65SBeau Belgrave * If we managed to get the page, re-issue the write. We do not
45481f8fb65SBeau Belgrave * want to get into a possible infinite loop, which is why we only
45581f8fb65SBeau Belgrave * attempt again directly if the page came in. If we couldn't get
45681f8fb65SBeau Belgrave * the page here, then we will try again the next time the event is
45781f8fb65SBeau Belgrave * enabled/disabled.
45881f8fb65SBeau Belgrave */
45981f8fb65SBeau Belgrave clear_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler));
46081f8fb65SBeau Belgrave
46181f8fb65SBeau Belgrave if (!ret) {
46281f8fb65SBeau Belgrave mmap_read_lock(mm->mm);
46341d8fba1SBeau Belgrave user_event_enabler_write(mm, enabler, true, &attempt);
46481f8fb65SBeau Belgrave mmap_read_unlock(mm->mm);
46581f8fb65SBeau Belgrave }
466dcb8177cSBeau Belgrave out:
46781f8fb65SBeau Belgrave mutex_unlock(&event_mutex);
46881f8fb65SBeau Belgrave
46981f8fb65SBeau Belgrave /* In all cases we no longer need the mm or fault */
47081f8fb65SBeau Belgrave user_event_mm_put(mm);
47181f8fb65SBeau Belgrave kmem_cache_free(fault_cache, fault);
47281f8fb65SBeau Belgrave }
47381f8fb65SBeau Belgrave
user_event_enabler_queue_fault(struct user_event_mm * mm,struct user_event_enabler * enabler,int attempt)47481f8fb65SBeau Belgrave static bool user_event_enabler_queue_fault(struct user_event_mm *mm,
47541d8fba1SBeau Belgrave struct user_event_enabler *enabler,
47641d8fba1SBeau Belgrave int attempt)
47772357590SBeau Belgrave {
47881f8fb65SBeau Belgrave struct user_event_enabler_fault *fault;
47981f8fb65SBeau Belgrave
48081f8fb65SBeau Belgrave fault = kmem_cache_zalloc(fault_cache, GFP_NOWAIT | __GFP_NOWARN);
48181f8fb65SBeau Belgrave
48281f8fb65SBeau Belgrave if (!fault)
48381f8fb65SBeau Belgrave return false;
48481f8fb65SBeau Belgrave
48581f8fb65SBeau Belgrave INIT_WORK(&fault->work, user_event_enabler_fault_fixup);
48681f8fb65SBeau Belgrave fault->mm = user_event_mm_get(mm);
48781f8fb65SBeau Belgrave fault->enabler = enabler;
48841d8fba1SBeau Belgrave fault->attempt = attempt;
48981f8fb65SBeau Belgrave
49081f8fb65SBeau Belgrave /* Don't try to queue in again while we have a pending fault */
49181f8fb65SBeau Belgrave set_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler));
49281f8fb65SBeau Belgrave
49381f8fb65SBeau Belgrave if (!schedule_work(&fault->work)) {
49481f8fb65SBeau Belgrave /* Allow another attempt later */
49581f8fb65SBeau Belgrave clear_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler));
49681f8fb65SBeau Belgrave
49781f8fb65SBeau Belgrave user_event_mm_put(mm);
49881f8fb65SBeau Belgrave kmem_cache_free(fault_cache, fault);
49981f8fb65SBeau Belgrave
50081f8fb65SBeau Belgrave return false;
50181f8fb65SBeau Belgrave }
50281f8fb65SBeau Belgrave
50381f8fb65SBeau Belgrave return true;
50481f8fb65SBeau Belgrave }
50581f8fb65SBeau Belgrave
user_event_enabler_write(struct user_event_mm * mm,struct user_event_enabler * enabler,bool fixup_fault,int * attempt)50681f8fb65SBeau Belgrave static int user_event_enabler_write(struct user_event_mm *mm,
50781f8fb65SBeau Belgrave struct user_event_enabler *enabler,
50841d8fba1SBeau Belgrave bool fixup_fault, int *attempt)
50981f8fb65SBeau Belgrave {
51072357590SBeau Belgrave unsigned long uaddr = enabler->addr;
51172357590SBeau Belgrave unsigned long *ptr;
51272357590SBeau Belgrave struct page *page;
51372357590SBeau Belgrave void *kaddr;
514*2de9ee94SBeau Belgrave int bit = ENABLE_BIT(enabler);
51572357590SBeau Belgrave int ret;
51672357590SBeau Belgrave
51772357590SBeau Belgrave lockdep_assert_held(&event_mutex);
51872357590SBeau Belgrave mmap_assert_locked(mm->mm);
51972357590SBeau Belgrave
52041d8fba1SBeau Belgrave *attempt += 1;
52141d8fba1SBeau Belgrave
52272357590SBeau Belgrave /* Ensure MM has tasks, cannot use after exit_mm() */
52372357590SBeau Belgrave if (refcount_read(&mm->tasks) == 0)
52472357590SBeau Belgrave return -ENOENT;
52572357590SBeau Belgrave
526dcb8177cSBeau Belgrave if (unlikely(test_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler)) ||
527dcb8177cSBeau Belgrave test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler))))
52881f8fb65SBeau Belgrave return -EBUSY;
52981f8fb65SBeau Belgrave
530*2de9ee94SBeau Belgrave align_addr_bit(&uaddr, &bit, ENABLE_BITOPS(enabler));
531*2de9ee94SBeau Belgrave
53272357590SBeau Belgrave ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT,
5330b295316SLorenzo Stoakes &page, NULL);
53472357590SBeau Belgrave
53581f8fb65SBeau Belgrave if (unlikely(ret <= 0)) {
53681f8fb65SBeau Belgrave if (!fixup_fault)
53781f8fb65SBeau Belgrave return -EFAULT;
53881f8fb65SBeau Belgrave
53941d8fba1SBeau Belgrave if (!user_event_enabler_queue_fault(mm, enabler, *attempt))
54081f8fb65SBeau Belgrave pr_warn("user_events: Unable to queue fault handler\n");
54181f8fb65SBeau Belgrave
54272357590SBeau Belgrave return -EFAULT;
54372357590SBeau Belgrave }
54472357590SBeau Belgrave
54572357590SBeau Belgrave kaddr = kmap_local_page(page);
54672357590SBeau Belgrave ptr = kaddr + (uaddr & ~PAGE_MASK);
54772357590SBeau Belgrave
54872357590SBeau Belgrave /* Update bit atomically, user tracers must be atomic as well */
54972357590SBeau Belgrave if (enabler->event && enabler->event->status)
550*2de9ee94SBeau Belgrave set_bit(bit, ptr);
55172357590SBeau Belgrave else
552*2de9ee94SBeau Belgrave clear_bit(bit, ptr);
55372357590SBeau Belgrave
55472357590SBeau Belgrave kunmap_local(kaddr);
55572357590SBeau Belgrave unpin_user_pages_dirty_lock(&page, 1, true);
55672357590SBeau Belgrave
55772357590SBeau Belgrave return 0;
55872357590SBeau Belgrave }
55972357590SBeau Belgrave
user_event_enabler_exists(struct user_event_mm * mm,unsigned long uaddr,unsigned char bit)56097bbce89SBeau Belgrave static bool user_event_enabler_exists(struct user_event_mm *mm,
56197bbce89SBeau Belgrave unsigned long uaddr, unsigned char bit)
56297bbce89SBeau Belgrave {
56397bbce89SBeau Belgrave struct user_event_enabler *enabler;
56497bbce89SBeau Belgrave
565dcbd1ac2SBeau Belgrave list_for_each_entry(enabler, &mm->enablers, mm_enablers_link) {
566ee7751b5SBeau Belgrave if (enabler->addr == uaddr && ENABLE_BIT(enabler) == bit)
56797bbce89SBeau Belgrave return true;
56897bbce89SBeau Belgrave }
56997bbce89SBeau Belgrave
57097bbce89SBeau Belgrave return false;
57197bbce89SBeau Belgrave }
57297bbce89SBeau Belgrave
user_event_enabler_update(struct user_event * user)57372357590SBeau Belgrave static void user_event_enabler_update(struct user_event *user)
57472357590SBeau Belgrave {
57572357590SBeau Belgrave struct user_event_enabler *enabler;
57672357590SBeau Belgrave struct user_event_mm *next;
577ff9e1632SBeau Belgrave struct user_event_mm *mm;
57841d8fba1SBeau Belgrave int attempt;
57972357590SBeau Belgrave
580aaecdaf9SLinus Torvalds lockdep_assert_held(&event_mutex);
581aaecdaf9SLinus Torvalds
582ff9e1632SBeau Belgrave /*
583ff9e1632SBeau Belgrave * We need to build a one-shot list of all the mms that have an
584ff9e1632SBeau Belgrave * enabler for the user_event passed in. This list is only valid
585ff9e1632SBeau Belgrave * while holding the event_mutex. The only reason for this is due
586ff9e1632SBeau Belgrave * to the global mm list being RCU protected and we use methods
587ff9e1632SBeau Belgrave * which can wait (mmap_read_lock and pin_user_pages_remote).
588ff9e1632SBeau Belgrave *
589ff9e1632SBeau Belgrave * NOTE: user_event_mm_get_all() increments the ref count of each
590ff9e1632SBeau Belgrave * mm that is added to the list to prevent removal timing windows.
591ff9e1632SBeau Belgrave * We must always put each mm after they are used, which may wait.
592ff9e1632SBeau Belgrave */
593ff9e1632SBeau Belgrave mm = user_event_mm_get_all(user);
594ff9e1632SBeau Belgrave
59572357590SBeau Belgrave while (mm) {
59672357590SBeau Belgrave next = mm->next;
59772357590SBeau Belgrave mmap_read_lock(mm->mm);
59872357590SBeau Belgrave
599dcbd1ac2SBeau Belgrave list_for_each_entry(enabler, &mm->enablers, mm_enablers_link) {
60041d8fba1SBeau Belgrave if (enabler->event == user) {
60141d8fba1SBeau Belgrave attempt = 0;
60241d8fba1SBeau Belgrave user_event_enabler_write(mm, enabler, true, &attempt);
60341d8fba1SBeau Belgrave }
60441d8fba1SBeau Belgrave }
60572357590SBeau Belgrave
60672357590SBeau Belgrave mmap_read_unlock(mm->mm);
60772357590SBeau Belgrave user_event_mm_put(mm);
60872357590SBeau Belgrave mm = next;
60972357590SBeau Belgrave }
61072357590SBeau Belgrave }
61172357590SBeau Belgrave
user_event_enabler_dup(struct user_event_enabler * orig,struct user_event_mm * mm)61272357590SBeau Belgrave static bool user_event_enabler_dup(struct user_event_enabler *orig,
61372357590SBeau Belgrave struct user_event_mm *mm)
61472357590SBeau Belgrave {
61572357590SBeau Belgrave struct user_event_enabler *enabler;
61672357590SBeau Belgrave
617dcb8177cSBeau Belgrave /* Skip pending frees */
618dcb8177cSBeau Belgrave if (unlikely(test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(orig))))
619dcb8177cSBeau Belgrave return true;
620dcb8177cSBeau Belgrave
621f9cce238SBeau Belgrave enabler = kzalloc(sizeof(*enabler), GFP_NOWAIT | __GFP_ACCOUNT);
62272357590SBeau Belgrave
62372357590SBeau Belgrave if (!enabler)
62472357590SBeau Belgrave return false;
62572357590SBeau Belgrave
626f0dbf6fdSBeau Belgrave enabler->event = user_event_get(orig->event);
62772357590SBeau Belgrave enabler->addr = orig->addr;
62872357590SBeau Belgrave
62972357590SBeau Belgrave /* Only dup part of value (ignore future flags, etc) */
63072357590SBeau Belgrave enabler->values = orig->values & ENABLE_VAL_DUP_MASK;
63172357590SBeau Belgrave
632aaecdaf9SLinus Torvalds /* Enablers not exposed yet, RCU not required */
633dcbd1ac2SBeau Belgrave list_add(&enabler->mm_enablers_link, &mm->enablers);
63472357590SBeau Belgrave
63572357590SBeau Belgrave return true;
63672357590SBeau Belgrave }
63772357590SBeau Belgrave
user_event_mm_get(struct user_event_mm * mm)63872357590SBeau Belgrave static struct user_event_mm *user_event_mm_get(struct user_event_mm *mm)
63972357590SBeau Belgrave {
64072357590SBeau Belgrave refcount_inc(&mm->refcnt);
64172357590SBeau Belgrave
64272357590SBeau Belgrave return mm;
64372357590SBeau Belgrave }
64472357590SBeau Belgrave
user_event_mm_get_all(struct user_event * user)64572357590SBeau Belgrave static struct user_event_mm *user_event_mm_get_all(struct user_event *user)
64672357590SBeau Belgrave {
64772357590SBeau Belgrave struct user_event_mm *found = NULL;
64872357590SBeau Belgrave struct user_event_enabler *enabler;
64972357590SBeau Belgrave struct user_event_mm *mm;
65072357590SBeau Belgrave
65172357590SBeau Belgrave /*
652ff9e1632SBeau Belgrave * We use the mm->next field to build a one-shot list from the global
653ff9e1632SBeau Belgrave * RCU protected list. To build this list the event_mutex must be held.
654ff9e1632SBeau Belgrave * This lets us build a list without requiring allocs that could fail
655ff9e1632SBeau Belgrave * when user based events are most wanted for diagnostics.
656ff9e1632SBeau Belgrave */
657ff9e1632SBeau Belgrave lockdep_assert_held(&event_mutex);
658ff9e1632SBeau Belgrave
659ff9e1632SBeau Belgrave /*
66072357590SBeau Belgrave * We do not want to block fork/exec while enablements are being
66172357590SBeau Belgrave * updated, so we use RCU to walk the current tasks that have used
66272357590SBeau Belgrave * user_events ABI for 1 or more events. Each enabler found in each
66372357590SBeau Belgrave * task that matches the event being updated has a write to reflect
66472357590SBeau Belgrave * the kernel state back into the process. Waits/faults must not occur
66572357590SBeau Belgrave * during this. So we scan the list under RCU for all the mm that have
66672357590SBeau Belgrave * the event within it. This is needed because mm_read_lock() can wait.
66772357590SBeau Belgrave * Each user mm returned has a ref inc to handle remove RCU races.
66872357590SBeau Belgrave */
66972357590SBeau Belgrave rcu_read_lock();
67072357590SBeau Belgrave
671dcbd1ac2SBeau Belgrave list_for_each_entry_rcu(mm, &user_event_mms, mms_link) {
672dcbd1ac2SBeau Belgrave list_for_each_entry_rcu(enabler, &mm->enablers, mm_enablers_link) {
67372357590SBeau Belgrave if (enabler->event == user) {
67472357590SBeau Belgrave mm->next = found;
67572357590SBeau Belgrave found = user_event_mm_get(mm);
67672357590SBeau Belgrave break;
67772357590SBeau Belgrave }
678dcbd1ac2SBeau Belgrave }
679dcbd1ac2SBeau Belgrave }
68072357590SBeau Belgrave
68172357590SBeau Belgrave rcu_read_unlock();
68272357590SBeau Belgrave
68372357590SBeau Belgrave return found;
68472357590SBeau Belgrave }
68572357590SBeau Belgrave
user_event_mm_alloc(struct task_struct * t)6863e0fea09SLinus Torvalds static struct user_event_mm *user_event_mm_alloc(struct task_struct *t)
68772357590SBeau Belgrave {
68872357590SBeau Belgrave struct user_event_mm *user_mm;
68972357590SBeau Belgrave
690f9cce238SBeau Belgrave user_mm = kzalloc(sizeof(*user_mm), GFP_KERNEL_ACCOUNT);
69172357590SBeau Belgrave
69272357590SBeau Belgrave if (!user_mm)
69372357590SBeau Belgrave return NULL;
69472357590SBeau Belgrave
69572357590SBeau Belgrave user_mm->mm = t->mm;
69672357590SBeau Belgrave INIT_LIST_HEAD(&user_mm->enablers);
69772357590SBeau Belgrave refcount_set(&user_mm->refcnt, 1);
69872357590SBeau Belgrave refcount_set(&user_mm->tasks, 1);
69972357590SBeau Belgrave
70072357590SBeau Belgrave /*
70172357590SBeau Belgrave * The lifetime of the memory descriptor can slightly outlast
70272357590SBeau Belgrave * the task lifetime if a ref to the user_event_mm is taken
70372357590SBeau Belgrave * between list_del_rcu() and call_rcu(). Therefore we need
70472357590SBeau Belgrave * to take a reference to it to ensure it can live this long
70572357590SBeau Belgrave * under this corner case. This can also occur in clones that
70672357590SBeau Belgrave * outlast the parent.
70772357590SBeau Belgrave */
70872357590SBeau Belgrave mmgrab(user_mm->mm);
70972357590SBeau Belgrave
71072357590SBeau Belgrave return user_mm;
71172357590SBeau Belgrave }
71272357590SBeau Belgrave
user_event_mm_attach(struct user_event_mm * user_mm,struct task_struct * t)7133e0fea09SLinus Torvalds static void user_event_mm_attach(struct user_event_mm *user_mm, struct task_struct *t)
7143e0fea09SLinus Torvalds {
7153e0fea09SLinus Torvalds unsigned long flags;
7163e0fea09SLinus Torvalds
7173e0fea09SLinus Torvalds spin_lock_irqsave(&user_event_mms_lock, flags);
718dcbd1ac2SBeau Belgrave list_add_rcu(&user_mm->mms_link, &user_event_mms);
7193e0fea09SLinus Torvalds spin_unlock_irqrestore(&user_event_mms_lock, flags);
7203e0fea09SLinus Torvalds
7213e0fea09SLinus Torvalds t->user_event_mm = user_mm;
7223e0fea09SLinus Torvalds }
7233e0fea09SLinus Torvalds
current_user_event_mm(void)72472357590SBeau Belgrave static struct user_event_mm *current_user_event_mm(void)
72572357590SBeau Belgrave {
72672357590SBeau Belgrave struct user_event_mm *user_mm = current->user_event_mm;
72772357590SBeau Belgrave
72872357590SBeau Belgrave if (user_mm)
72972357590SBeau Belgrave goto inc;
73072357590SBeau Belgrave
7313e0fea09SLinus Torvalds user_mm = user_event_mm_alloc(current);
73272357590SBeau Belgrave
73372357590SBeau Belgrave if (!user_mm)
73472357590SBeau Belgrave goto error;
7353e0fea09SLinus Torvalds
7363e0fea09SLinus Torvalds user_event_mm_attach(user_mm, current);
73772357590SBeau Belgrave inc:
73872357590SBeau Belgrave refcount_inc(&user_mm->refcnt);
73972357590SBeau Belgrave error:
74072357590SBeau Belgrave return user_mm;
74172357590SBeau Belgrave }
74272357590SBeau Belgrave
user_event_mm_destroy(struct user_event_mm * mm)74372357590SBeau Belgrave static void user_event_mm_destroy(struct user_event_mm *mm)
74472357590SBeau Belgrave {
74572357590SBeau Belgrave struct user_event_enabler *enabler, *next;
74672357590SBeau Belgrave
747dcbd1ac2SBeau Belgrave list_for_each_entry_safe(enabler, next, &mm->enablers, mm_enablers_link)
748f0dbf6fdSBeau Belgrave user_event_enabler_destroy(enabler, false);
74972357590SBeau Belgrave
75072357590SBeau Belgrave mmdrop(mm->mm);
75172357590SBeau Belgrave kfree(mm);
75272357590SBeau Belgrave }
75372357590SBeau Belgrave
user_event_mm_put(struct user_event_mm * mm)75472357590SBeau Belgrave static void user_event_mm_put(struct user_event_mm *mm)
75572357590SBeau Belgrave {
75672357590SBeau Belgrave if (mm && refcount_dec_and_test(&mm->refcnt))
75772357590SBeau Belgrave user_event_mm_destroy(mm);
75872357590SBeau Belgrave }
75972357590SBeau Belgrave
delayed_user_event_mm_put(struct work_struct * work)76072357590SBeau Belgrave static void delayed_user_event_mm_put(struct work_struct *work)
76172357590SBeau Belgrave {
76272357590SBeau Belgrave struct user_event_mm *mm;
76372357590SBeau Belgrave
76472357590SBeau Belgrave mm = container_of(to_rcu_work(work), struct user_event_mm, put_rwork);
76572357590SBeau Belgrave user_event_mm_put(mm);
76672357590SBeau Belgrave }
76772357590SBeau Belgrave
user_event_mm_remove(struct task_struct * t)76872357590SBeau Belgrave void user_event_mm_remove(struct task_struct *t)
76972357590SBeau Belgrave {
77072357590SBeau Belgrave struct user_event_mm *mm;
77172357590SBeau Belgrave unsigned long flags;
77272357590SBeau Belgrave
77372357590SBeau Belgrave might_sleep();
77472357590SBeau Belgrave
77572357590SBeau Belgrave mm = t->user_event_mm;
77672357590SBeau Belgrave t->user_event_mm = NULL;
77772357590SBeau Belgrave
77872357590SBeau Belgrave /* Clone will increment the tasks, only remove if last clone */
77972357590SBeau Belgrave if (!refcount_dec_and_test(&mm->tasks))
78072357590SBeau Belgrave return;
78172357590SBeau Belgrave
78272357590SBeau Belgrave /* Remove the mm from the list, so it can no longer be enabled */
78372357590SBeau Belgrave spin_lock_irqsave(&user_event_mms_lock, flags);
784dcbd1ac2SBeau Belgrave list_del_rcu(&mm->mms_link);
78572357590SBeau Belgrave spin_unlock_irqrestore(&user_event_mms_lock, flags);
78672357590SBeau Belgrave
78772357590SBeau Belgrave /*
78872357590SBeau Belgrave * We need to wait for currently occurring writes to stop within
78972357590SBeau Belgrave * the mm. This is required since exit_mm() snaps the current rss
79072357590SBeau Belgrave * stats and clears them. On the final mmdrop(), check_mm() will
79172357590SBeau Belgrave * report a bug if these increment.
79272357590SBeau Belgrave *
79372357590SBeau Belgrave * All writes/pins are done under mmap_read lock, take the write
79472357590SBeau Belgrave * lock to ensure in-progress faults have completed. Faults that
79572357590SBeau Belgrave * are pending but yet to run will check the task count and skip
79672357590SBeau Belgrave * the fault since the mm is going away.
79772357590SBeau Belgrave */
79872357590SBeau Belgrave mmap_write_lock(mm->mm);
79972357590SBeau Belgrave mmap_write_unlock(mm->mm);
80072357590SBeau Belgrave
80172357590SBeau Belgrave /*
80272357590SBeau Belgrave * Put for mm must be done after RCU delay to handle new refs in
80372357590SBeau Belgrave * between the list_del_rcu() and now. This ensures any get refs
80472357590SBeau Belgrave * during rcu_read_lock() are accounted for during list removal.
80572357590SBeau Belgrave *
80672357590SBeau Belgrave * CPU A | CPU B
80772357590SBeau Belgrave * ---------------------------------------------------------------
80872357590SBeau Belgrave * user_event_mm_remove() | rcu_read_lock();
80972357590SBeau Belgrave * list_del_rcu() | list_for_each_entry_rcu();
81072357590SBeau Belgrave * call_rcu() | refcount_inc();
81172357590SBeau Belgrave * . | rcu_read_unlock();
81272357590SBeau Belgrave * schedule_work() | .
81372357590SBeau Belgrave * user_event_mm_put() | .
81472357590SBeau Belgrave *
81572357590SBeau Belgrave * mmdrop() cannot be called in the softirq context of call_rcu()
81672357590SBeau Belgrave * so we use a work queue after call_rcu() to run within.
81772357590SBeau Belgrave */
81872357590SBeau Belgrave INIT_RCU_WORK(&mm->put_rwork, delayed_user_event_mm_put);
81972357590SBeau Belgrave queue_rcu_work(system_wq, &mm->put_rwork);
82072357590SBeau Belgrave }
82172357590SBeau Belgrave
user_event_mm_dup(struct task_struct * t,struct user_event_mm * old_mm)82272357590SBeau Belgrave void user_event_mm_dup(struct task_struct *t, struct user_event_mm *old_mm)
82372357590SBeau Belgrave {
8243e0fea09SLinus Torvalds struct user_event_mm *mm = user_event_mm_alloc(t);
82572357590SBeau Belgrave struct user_event_enabler *enabler;
82672357590SBeau Belgrave
82772357590SBeau Belgrave if (!mm)
82872357590SBeau Belgrave return;
82972357590SBeau Belgrave
83072357590SBeau Belgrave rcu_read_lock();
83172357590SBeau Belgrave
832dcbd1ac2SBeau Belgrave list_for_each_entry_rcu(enabler, &old_mm->enablers, mm_enablers_link) {
83372357590SBeau Belgrave if (!user_event_enabler_dup(enabler, mm))
83472357590SBeau Belgrave goto error;
835dcbd1ac2SBeau Belgrave }
83672357590SBeau Belgrave
83772357590SBeau Belgrave rcu_read_unlock();
83872357590SBeau Belgrave
8393e0fea09SLinus Torvalds user_event_mm_attach(mm, t);
84072357590SBeau Belgrave return;
84172357590SBeau Belgrave error:
84272357590SBeau Belgrave rcu_read_unlock();
8433e0fea09SLinus Torvalds user_event_mm_destroy(mm);
84472357590SBeau Belgrave }
84572357590SBeau Belgrave
current_user_event_enabler_exists(unsigned long uaddr,unsigned char bit)84697bbce89SBeau Belgrave static bool current_user_event_enabler_exists(unsigned long uaddr,
84797bbce89SBeau Belgrave unsigned char bit)
84897bbce89SBeau Belgrave {
84997bbce89SBeau Belgrave struct user_event_mm *user_mm = current_user_event_mm();
85097bbce89SBeau Belgrave bool exists;
85197bbce89SBeau Belgrave
85297bbce89SBeau Belgrave if (!user_mm)
85397bbce89SBeau Belgrave return false;
85497bbce89SBeau Belgrave
85597bbce89SBeau Belgrave exists = user_event_enabler_exists(user_mm, uaddr, bit);
85697bbce89SBeau Belgrave
85797bbce89SBeau Belgrave user_event_mm_put(user_mm);
85897bbce89SBeau Belgrave
85997bbce89SBeau Belgrave return exists;
86097bbce89SBeau Belgrave }
86197bbce89SBeau Belgrave
86272357590SBeau Belgrave static struct user_event_enabler
user_event_enabler_create(struct user_reg * reg,struct user_event * user,int * write_result)86372357590SBeau Belgrave *user_event_enabler_create(struct user_reg *reg, struct user_event *user,
86472357590SBeau Belgrave int *write_result)
86572357590SBeau Belgrave {
86672357590SBeau Belgrave struct user_event_enabler *enabler;
86772357590SBeau Belgrave struct user_event_mm *user_mm;
86872357590SBeau Belgrave unsigned long uaddr = (unsigned long)reg->enable_addr;
86941d8fba1SBeau Belgrave int attempt = 0;
87072357590SBeau Belgrave
87172357590SBeau Belgrave user_mm = current_user_event_mm();
87272357590SBeau Belgrave
87372357590SBeau Belgrave if (!user_mm)
87472357590SBeau Belgrave return NULL;
87572357590SBeau Belgrave
876f9cce238SBeau Belgrave enabler = kzalloc(sizeof(*enabler), GFP_KERNEL_ACCOUNT);
87772357590SBeau Belgrave
87872357590SBeau Belgrave if (!enabler)
87972357590SBeau Belgrave goto out;
88072357590SBeau Belgrave
88172357590SBeau Belgrave enabler->event = user;
88272357590SBeau Belgrave enabler->addr = uaddr;
88372357590SBeau Belgrave enabler->values = reg->enable_bit;
884*2de9ee94SBeau Belgrave
885*2de9ee94SBeau Belgrave #if BITS_PER_LONG >= 64
886*2de9ee94SBeau Belgrave if (reg->enable_size == 4)
887*2de9ee94SBeau Belgrave set_bit(ENABLE_VAL_32_ON_64_BIT, ENABLE_BITOPS(enabler));
888*2de9ee94SBeau Belgrave #endif
889*2de9ee94SBeau Belgrave
89072357590SBeau Belgrave retry:
89172357590SBeau Belgrave /* Prevents state changes from racing with new enablers */
89272357590SBeau Belgrave mutex_lock(&event_mutex);
89372357590SBeau Belgrave
89472357590SBeau Belgrave /* Attempt to reflect the current state within the process */
89572357590SBeau Belgrave mmap_read_lock(user_mm->mm);
89641d8fba1SBeau Belgrave *write_result = user_event_enabler_write(user_mm, enabler, false,
89741d8fba1SBeau Belgrave &attempt);
89872357590SBeau Belgrave mmap_read_unlock(user_mm->mm);
89972357590SBeau Belgrave
90072357590SBeau Belgrave /*
90172357590SBeau Belgrave * If the write works, then we will track the enabler. A ref to the
90272357590SBeau Belgrave * underlying user_event is held by the enabler to prevent it going
90372357590SBeau Belgrave * away while the enabler is still in use by a process. The ref is
90472357590SBeau Belgrave * removed when the enabler is destroyed. This means a event cannot
90572357590SBeau Belgrave * be forcefully deleted from the system until all tasks using it
90672357590SBeau Belgrave * exit or run exec(), which includes forks and clones.
90772357590SBeau Belgrave */
90872357590SBeau Belgrave if (!*write_result) {
909f0dbf6fdSBeau Belgrave user_event_get(user);
910dcbd1ac2SBeau Belgrave list_add_rcu(&enabler->mm_enablers_link, &user_mm->enablers);
91172357590SBeau Belgrave }
91272357590SBeau Belgrave
91372357590SBeau Belgrave mutex_unlock(&event_mutex);
91472357590SBeau Belgrave
91572357590SBeau Belgrave if (*write_result) {
91672357590SBeau Belgrave /* Attempt to fault-in and retry if it worked */
91741d8fba1SBeau Belgrave if (!user_event_mm_fault_in(user_mm, uaddr, attempt))
91872357590SBeau Belgrave goto retry;
91972357590SBeau Belgrave
92072357590SBeau Belgrave kfree(enabler);
92172357590SBeau Belgrave enabler = NULL;
92272357590SBeau Belgrave }
92372357590SBeau Belgrave out:
92472357590SBeau Belgrave user_event_mm_put(user_mm);
92572357590SBeau Belgrave
92672357590SBeau Belgrave return enabler;
92739d6d08bSBeau Belgrave }
92839d6d08bSBeau Belgrave
9290279400aSBeau Belgrave static __always_inline __must_check
user_event_last_ref(struct user_event * user)930d401b724SBeau Belgrave bool user_event_last_ref(struct user_event *user)
931d401b724SBeau Belgrave {
932a65442edSBeau Belgrave int last = 0;
933a65442edSBeau Belgrave
934a65442edSBeau Belgrave if (user->reg_flags & USER_EVENT_REG_PERSIST)
935a65442edSBeau Belgrave last = 1;
936a65442edSBeau Belgrave
937a65442edSBeau Belgrave return refcount_read(&user->refcnt) == last;
938d401b724SBeau Belgrave }
939d401b724SBeau Belgrave
940d401b724SBeau Belgrave static __always_inline __must_check
copy_nofault(void * addr,size_t bytes,struct iov_iter * i)9410279400aSBeau Belgrave size_t copy_nofault(void *addr, size_t bytes, struct iov_iter *i)
9420279400aSBeau Belgrave {
9430279400aSBeau Belgrave size_t ret;
9440279400aSBeau Belgrave
9450279400aSBeau Belgrave pagefault_disable();
9460279400aSBeau Belgrave
9470279400aSBeau Belgrave ret = copy_from_iter_nocache(addr, bytes, i);
9480279400aSBeau Belgrave
9490279400aSBeau Belgrave pagefault_enable();
9500279400aSBeau Belgrave
9510279400aSBeau Belgrave return ret;
9520279400aSBeau Belgrave }
9530279400aSBeau Belgrave
user_event_get_fields(struct trace_event_call * call)9547f5a08c7SBeau Belgrave static struct list_head *user_event_get_fields(struct trace_event_call *call)
9557f5a08c7SBeau Belgrave {
9567f5a08c7SBeau Belgrave struct user_event *user = (struct user_event *)call->data;
9577f5a08c7SBeau Belgrave
9587f5a08c7SBeau Belgrave return &user->fields;
9597f5a08c7SBeau Belgrave }
9607f5a08c7SBeau Belgrave
9617f5a08c7SBeau Belgrave /*
9627f5a08c7SBeau Belgrave * Parses a register command for user_events
9637f5a08c7SBeau Belgrave * Format: event_name[:FLAG1[,FLAG2...]] [field1[;field2...]]
9647f5a08c7SBeau Belgrave *
9657f5a08c7SBeau Belgrave * Example event named 'test' with a 20 char 'msg' field with an unsigned int
9667f5a08c7SBeau Belgrave * 'id' field after:
9677f5a08c7SBeau Belgrave * test char[20] msg;unsigned int id
9687f5a08c7SBeau Belgrave *
9697f5a08c7SBeau Belgrave * NOTE: Offsets are from the user data perspective, they are not from the
9707f5a08c7SBeau Belgrave * trace_entry/buffer perspective. We automatically add the common properties
9717f5a08c7SBeau Belgrave * sizes to the offset for the user.
9727e348b32SBeau Belgrave *
9737e348b32SBeau Belgrave * Upon success user_event has its ref count increased by 1.
9747f5a08c7SBeau Belgrave */
user_event_parse_cmd(struct user_event_group * group,char * raw_command,struct user_event ** newuser,int reg_flags)975e5d27181SBeau Belgrave static int user_event_parse_cmd(struct user_event_group *group,
976b08d7258SBeau Belgrave char *raw_command, struct user_event **newuser,
977b08d7258SBeau Belgrave int reg_flags)
9787f5a08c7SBeau Belgrave {
9797f5a08c7SBeau Belgrave char *name = raw_command;
9807f5a08c7SBeau Belgrave char *args = strpbrk(name, " ");
9817f5a08c7SBeau Belgrave char *flags;
9827f5a08c7SBeau Belgrave
9837f5a08c7SBeau Belgrave if (args)
9847f5a08c7SBeau Belgrave *args++ = '\0';
9857f5a08c7SBeau Belgrave
9867f5a08c7SBeau Belgrave flags = strpbrk(name, ":");
9877f5a08c7SBeau Belgrave
9887f5a08c7SBeau Belgrave if (flags)
9897f5a08c7SBeau Belgrave *flags++ = '\0';
9907f5a08c7SBeau Belgrave
991b08d7258SBeau Belgrave return user_event_parse(group, name, args, flags, newuser, reg_flags);
9927f5a08c7SBeau Belgrave }
9937f5a08c7SBeau Belgrave
user_field_array_size(const char * type)9947f5a08c7SBeau Belgrave static int user_field_array_size(const char *type)
9957f5a08c7SBeau Belgrave {
9967f5a08c7SBeau Belgrave const char *start = strchr(type, '[');
9977f5a08c7SBeau Belgrave char val[8];
9987f5a08c7SBeau Belgrave char *bracket;
9997f5a08c7SBeau Belgrave int size = 0;
10007f5a08c7SBeau Belgrave
10017f5a08c7SBeau Belgrave if (start == NULL)
10027f5a08c7SBeau Belgrave return -EINVAL;
10037f5a08c7SBeau Belgrave
10047f5a08c7SBeau Belgrave if (strscpy(val, start + 1, sizeof(val)) <= 0)
10057f5a08c7SBeau Belgrave return -EINVAL;
10067f5a08c7SBeau Belgrave
10077f5a08c7SBeau Belgrave bracket = strchr(val, ']');
10087f5a08c7SBeau Belgrave
10097f5a08c7SBeau Belgrave if (!bracket)
10107f5a08c7SBeau Belgrave return -EINVAL;
10117f5a08c7SBeau Belgrave
10127f5a08c7SBeau Belgrave *bracket = '\0';
10137f5a08c7SBeau Belgrave
10147f5a08c7SBeau Belgrave if (kstrtouint(val, 0, &size))
10157f5a08c7SBeau Belgrave return -EINVAL;
10167f5a08c7SBeau Belgrave
10177f5a08c7SBeau Belgrave if (size > MAX_FIELD_ARRAY_SIZE)
10187f5a08c7SBeau Belgrave return -EINVAL;
10197f5a08c7SBeau Belgrave
10207f5a08c7SBeau Belgrave return size;
10217f5a08c7SBeau Belgrave }
10227f5a08c7SBeau Belgrave
user_field_size(const char * type)10237f5a08c7SBeau Belgrave static int user_field_size(const char *type)
10247f5a08c7SBeau Belgrave {
10257f5a08c7SBeau Belgrave /* long is not allowed from a user, since it's ambigious in size */
10267f5a08c7SBeau Belgrave if (strcmp(type, "s64") == 0)
10277f5a08c7SBeau Belgrave return sizeof(s64);
10287f5a08c7SBeau Belgrave if (strcmp(type, "u64") == 0)
10297f5a08c7SBeau Belgrave return sizeof(u64);
10307f5a08c7SBeau Belgrave if (strcmp(type, "s32") == 0)
10317f5a08c7SBeau Belgrave return sizeof(s32);
10327f5a08c7SBeau Belgrave if (strcmp(type, "u32") == 0)
10337f5a08c7SBeau Belgrave return sizeof(u32);
10347f5a08c7SBeau Belgrave if (strcmp(type, "int") == 0)
10357f5a08c7SBeau Belgrave return sizeof(int);
10367f5a08c7SBeau Belgrave if (strcmp(type, "unsigned int") == 0)
10377f5a08c7SBeau Belgrave return sizeof(unsigned int);
10387f5a08c7SBeau Belgrave if (strcmp(type, "s16") == 0)
10397f5a08c7SBeau Belgrave return sizeof(s16);
10407f5a08c7SBeau Belgrave if (strcmp(type, "u16") == 0)
10417f5a08c7SBeau Belgrave return sizeof(u16);
10427f5a08c7SBeau Belgrave if (strcmp(type, "short") == 0)
10437f5a08c7SBeau Belgrave return sizeof(short);
10447f5a08c7SBeau Belgrave if (strcmp(type, "unsigned short") == 0)
10457f5a08c7SBeau Belgrave return sizeof(unsigned short);
10467f5a08c7SBeau Belgrave if (strcmp(type, "s8") == 0)
10477f5a08c7SBeau Belgrave return sizeof(s8);
10487f5a08c7SBeau Belgrave if (strcmp(type, "u8") == 0)
10497f5a08c7SBeau Belgrave return sizeof(u8);
10507f5a08c7SBeau Belgrave if (strcmp(type, "char") == 0)
10517f5a08c7SBeau Belgrave return sizeof(char);
10527f5a08c7SBeau Belgrave if (strcmp(type, "unsigned char") == 0)
10537f5a08c7SBeau Belgrave return sizeof(unsigned char);
10547f5a08c7SBeau Belgrave if (str_has_prefix(type, "char["))
10557f5a08c7SBeau Belgrave return user_field_array_size(type);
10567f5a08c7SBeau Belgrave if (str_has_prefix(type, "unsigned char["))
10577f5a08c7SBeau Belgrave return user_field_array_size(type);
10587f5a08c7SBeau Belgrave if (str_has_prefix(type, "__data_loc "))
10597f5a08c7SBeau Belgrave return sizeof(u32);
10607f5a08c7SBeau Belgrave if (str_has_prefix(type, "__rel_loc "))
10617f5a08c7SBeau Belgrave return sizeof(u32);
10627f5a08c7SBeau Belgrave
10637f5a08c7SBeau Belgrave /* Uknown basic type, error */
10647f5a08c7SBeau Belgrave return -EINVAL;
10657f5a08c7SBeau Belgrave }
10667f5a08c7SBeau Belgrave
user_event_destroy_validators(struct user_event * user)10672467cda1SBeau Belgrave static void user_event_destroy_validators(struct user_event *user)
10682467cda1SBeau Belgrave {
10692467cda1SBeau Belgrave struct user_event_validator *validator, *next;
10702467cda1SBeau Belgrave struct list_head *head = &user->validators;
10712467cda1SBeau Belgrave
1072dcbd1ac2SBeau Belgrave list_for_each_entry_safe(validator, next, head, user_event_link) {
1073dcbd1ac2SBeau Belgrave list_del(&validator->user_event_link);
10742467cda1SBeau Belgrave kfree(validator);
10752467cda1SBeau Belgrave }
10762467cda1SBeau Belgrave }
10772467cda1SBeau Belgrave
user_event_destroy_fields(struct user_event * user)10787f5a08c7SBeau Belgrave static void user_event_destroy_fields(struct user_event *user)
10797f5a08c7SBeau Belgrave {
10807f5a08c7SBeau Belgrave struct ftrace_event_field *field, *next;
10817f5a08c7SBeau Belgrave struct list_head *head = &user->fields;
10827f5a08c7SBeau Belgrave
10837f5a08c7SBeau Belgrave list_for_each_entry_safe(field, next, head, link) {
10847f5a08c7SBeau Belgrave list_del(&field->link);
10857f5a08c7SBeau Belgrave kfree(field);
10867f5a08c7SBeau Belgrave }
10877f5a08c7SBeau Belgrave }
10887f5a08c7SBeau Belgrave
user_event_add_field(struct user_event * user,const char * type,const char * name,int offset,int size,int is_signed,int filter_type)10897f5a08c7SBeau Belgrave static int user_event_add_field(struct user_event *user, const char *type,
10907f5a08c7SBeau Belgrave const char *name, int offset, int size,
10917f5a08c7SBeau Belgrave int is_signed, int filter_type)
10927f5a08c7SBeau Belgrave {
10932467cda1SBeau Belgrave struct user_event_validator *validator;
10947f5a08c7SBeau Belgrave struct ftrace_event_field *field;
10952467cda1SBeau Belgrave int validator_flags = 0;
10967f5a08c7SBeau Belgrave
1097f9cce238SBeau Belgrave field = kmalloc(sizeof(*field), GFP_KERNEL_ACCOUNT);
10987f5a08c7SBeau Belgrave
10997f5a08c7SBeau Belgrave if (!field)
11007f5a08c7SBeau Belgrave return -ENOMEM;
11017f5a08c7SBeau Belgrave
11022467cda1SBeau Belgrave if (str_has_prefix(type, "__data_loc "))
11032467cda1SBeau Belgrave goto add_validator;
11042467cda1SBeau Belgrave
11052467cda1SBeau Belgrave if (str_has_prefix(type, "__rel_loc ")) {
11062467cda1SBeau Belgrave validator_flags |= VALIDATOR_REL;
11072467cda1SBeau Belgrave goto add_validator;
11082467cda1SBeau Belgrave }
11092467cda1SBeau Belgrave
11102467cda1SBeau Belgrave goto add_field;
11112467cda1SBeau Belgrave
11122467cda1SBeau Belgrave add_validator:
11139cbf1234SBeau Belgrave if (strstr(type, "char") != NULL)
11142467cda1SBeau Belgrave validator_flags |= VALIDATOR_ENSURE_NULL;
11152467cda1SBeau Belgrave
1116f9cce238SBeau Belgrave validator = kmalloc(sizeof(*validator), GFP_KERNEL_ACCOUNT);
11172467cda1SBeau Belgrave
11182467cda1SBeau Belgrave if (!validator) {
11192467cda1SBeau Belgrave kfree(field);
11202467cda1SBeau Belgrave return -ENOMEM;
11212467cda1SBeau Belgrave }
11222467cda1SBeau Belgrave
11232467cda1SBeau Belgrave validator->flags = validator_flags;
11242467cda1SBeau Belgrave validator->offset = offset;
11252467cda1SBeau Belgrave
11262467cda1SBeau Belgrave /* Want sequential access when validating */
1127dcbd1ac2SBeau Belgrave list_add_tail(&validator->user_event_link, &user->validators);
11282467cda1SBeau Belgrave
11292467cda1SBeau Belgrave add_field:
11307f5a08c7SBeau Belgrave field->type = type;
11317f5a08c7SBeau Belgrave field->name = name;
11327f5a08c7SBeau Belgrave field->offset = offset;
11337f5a08c7SBeau Belgrave field->size = size;
11347f5a08c7SBeau Belgrave field->is_signed = is_signed;
11357f5a08c7SBeau Belgrave field->filter_type = filter_type;
11367f5a08c7SBeau Belgrave
11379872c07bSBeau Belgrave if (filter_type == FILTER_OTHER)
11389872c07bSBeau Belgrave field->filter_type = filter_assign_type(type);
11399872c07bSBeau Belgrave
11407f5a08c7SBeau Belgrave list_add(&field->link, &user->fields);
11417f5a08c7SBeau Belgrave
11422467cda1SBeau Belgrave /*
11432467cda1SBeau Belgrave * Min size from user writes that are required, this does not include
11442467cda1SBeau Belgrave * the size of trace_entry (common fields).
11452467cda1SBeau Belgrave */
11462467cda1SBeau Belgrave user->min_size = (offset + size) - sizeof(struct trace_entry);
11472467cda1SBeau Belgrave
11487f5a08c7SBeau Belgrave return 0;
11497f5a08c7SBeau Belgrave }
11507f5a08c7SBeau Belgrave
11517f5a08c7SBeau Belgrave /*
11527f5a08c7SBeau Belgrave * Parses the values of a field within the description
11537f5a08c7SBeau Belgrave * Format: type name [size]
11547f5a08c7SBeau Belgrave */
user_event_parse_field(char * field,struct user_event * user,u32 * offset)11557f5a08c7SBeau Belgrave static int user_event_parse_field(char *field, struct user_event *user,
11567f5a08c7SBeau Belgrave u32 *offset)
11577f5a08c7SBeau Belgrave {
11587f5a08c7SBeau Belgrave char *part, *type, *name;
11597f5a08c7SBeau Belgrave u32 depth = 0, saved_offset = *offset;
11607f5a08c7SBeau Belgrave int len, size = -EINVAL;
11617f5a08c7SBeau Belgrave bool is_struct = false;
11627f5a08c7SBeau Belgrave
11637f5a08c7SBeau Belgrave field = skip_spaces(field);
11647f5a08c7SBeau Belgrave
11657f5a08c7SBeau Belgrave if (*field == '\0')
11667f5a08c7SBeau Belgrave return 0;
11677f5a08c7SBeau Belgrave
11687f5a08c7SBeau Belgrave /* Handle types that have a space within */
11697f5a08c7SBeau Belgrave len = str_has_prefix(field, "unsigned ");
11707f5a08c7SBeau Belgrave if (len)
11717f5a08c7SBeau Belgrave goto skip_next;
11727f5a08c7SBeau Belgrave
11737f5a08c7SBeau Belgrave len = str_has_prefix(field, "struct ");
11747f5a08c7SBeau Belgrave if (len) {
11757f5a08c7SBeau Belgrave is_struct = true;
11767f5a08c7SBeau Belgrave goto skip_next;
11777f5a08c7SBeau Belgrave }
11787f5a08c7SBeau Belgrave
11797f5a08c7SBeau Belgrave len = str_has_prefix(field, "__data_loc unsigned ");
11807f5a08c7SBeau Belgrave if (len)
11817f5a08c7SBeau Belgrave goto skip_next;
11827f5a08c7SBeau Belgrave
11837f5a08c7SBeau Belgrave len = str_has_prefix(field, "__data_loc ");
11847f5a08c7SBeau Belgrave if (len)
11857f5a08c7SBeau Belgrave goto skip_next;
11867f5a08c7SBeau Belgrave
11877f5a08c7SBeau Belgrave len = str_has_prefix(field, "__rel_loc unsigned ");
11887f5a08c7SBeau Belgrave if (len)
11897f5a08c7SBeau Belgrave goto skip_next;
11907f5a08c7SBeau Belgrave
11917f5a08c7SBeau Belgrave len = str_has_prefix(field, "__rel_loc ");
11927f5a08c7SBeau Belgrave if (len)
11937f5a08c7SBeau Belgrave goto skip_next;
11947f5a08c7SBeau Belgrave
11957f5a08c7SBeau Belgrave goto parse;
11967f5a08c7SBeau Belgrave skip_next:
11977f5a08c7SBeau Belgrave type = field;
11987f5a08c7SBeau Belgrave field = strpbrk(field + len, " ");
11997f5a08c7SBeau Belgrave
12007f5a08c7SBeau Belgrave if (field == NULL)
12017f5a08c7SBeau Belgrave return -EINVAL;
12027f5a08c7SBeau Belgrave
12037f5a08c7SBeau Belgrave *field++ = '\0';
12047f5a08c7SBeau Belgrave depth++;
12057f5a08c7SBeau Belgrave parse:
1206173c2049SBeau Belgrave name = NULL;
1207173c2049SBeau Belgrave
12087f5a08c7SBeau Belgrave while ((part = strsep(&field, " ")) != NULL) {
12097f5a08c7SBeau Belgrave switch (depth++) {
12107f5a08c7SBeau Belgrave case FIELD_DEPTH_TYPE:
12117f5a08c7SBeau Belgrave type = part;
12127f5a08c7SBeau Belgrave break;
12137f5a08c7SBeau Belgrave case FIELD_DEPTH_NAME:
12147f5a08c7SBeau Belgrave name = part;
12157f5a08c7SBeau Belgrave break;
12167f5a08c7SBeau Belgrave case FIELD_DEPTH_SIZE:
12177f5a08c7SBeau Belgrave if (!is_struct)
12187f5a08c7SBeau Belgrave return -EINVAL;
12197f5a08c7SBeau Belgrave
12207f5a08c7SBeau Belgrave if (kstrtou32(part, 10, &size))
12217f5a08c7SBeau Belgrave return -EINVAL;
12227f5a08c7SBeau Belgrave break;
12237f5a08c7SBeau Belgrave default:
12247f5a08c7SBeau Belgrave return -EINVAL;
12257f5a08c7SBeau Belgrave }
12267f5a08c7SBeau Belgrave }
12277f5a08c7SBeau Belgrave
1228173c2049SBeau Belgrave if (depth < FIELD_DEPTH_SIZE || !name)
12297f5a08c7SBeau Belgrave return -EINVAL;
12307f5a08c7SBeau Belgrave
12317f5a08c7SBeau Belgrave if (depth == FIELD_DEPTH_SIZE)
12327f5a08c7SBeau Belgrave size = user_field_size(type);
12337f5a08c7SBeau Belgrave
12347f5a08c7SBeau Belgrave if (size == 0)
12357f5a08c7SBeau Belgrave return -EINVAL;
12367f5a08c7SBeau Belgrave
12377f5a08c7SBeau Belgrave if (size < 0)
12387f5a08c7SBeau Belgrave return size;
12397f5a08c7SBeau Belgrave
12407f5a08c7SBeau Belgrave *offset = saved_offset + size;
12417f5a08c7SBeau Belgrave
12427f5a08c7SBeau Belgrave return user_event_add_field(user, type, name, saved_offset, size,
12437f5a08c7SBeau Belgrave type[0] != 'u', FILTER_OTHER);
12447f5a08c7SBeau Belgrave }
12457f5a08c7SBeau Belgrave
user_event_parse_fields(struct user_event * user,char * args)12467f5a08c7SBeau Belgrave static int user_event_parse_fields(struct user_event *user, char *args)
12477f5a08c7SBeau Belgrave {
12487f5a08c7SBeau Belgrave char *field;
12497f5a08c7SBeau Belgrave u32 offset = sizeof(struct trace_entry);
12507f5a08c7SBeau Belgrave int ret = -EINVAL;
12517f5a08c7SBeau Belgrave
12527f5a08c7SBeau Belgrave if (args == NULL)
12537f5a08c7SBeau Belgrave return 0;
12547f5a08c7SBeau Belgrave
12557f5a08c7SBeau Belgrave while ((field = strsep(&args, ";")) != NULL) {
12567f5a08c7SBeau Belgrave ret = user_event_parse_field(field, user, &offset);
12577f5a08c7SBeau Belgrave
12587f5a08c7SBeau Belgrave if (ret)
12597f5a08c7SBeau Belgrave break;
12607f5a08c7SBeau Belgrave }
12617f5a08c7SBeau Belgrave
12627f5a08c7SBeau Belgrave return ret;
12637f5a08c7SBeau Belgrave }
12647f5a08c7SBeau Belgrave
12657f5a08c7SBeau Belgrave static struct trace_event_fields user_event_fields_array[1];
12667f5a08c7SBeau Belgrave
user_field_format(const char * type)1267aa3b2b4cSBeau Belgrave static const char *user_field_format(const char *type)
1268aa3b2b4cSBeau Belgrave {
1269aa3b2b4cSBeau Belgrave if (strcmp(type, "s64") == 0)
1270aa3b2b4cSBeau Belgrave return "%lld";
1271aa3b2b4cSBeau Belgrave if (strcmp(type, "u64") == 0)
1272aa3b2b4cSBeau Belgrave return "%llu";
1273aa3b2b4cSBeau Belgrave if (strcmp(type, "s32") == 0)
1274aa3b2b4cSBeau Belgrave return "%d";
1275aa3b2b4cSBeau Belgrave if (strcmp(type, "u32") == 0)
1276aa3b2b4cSBeau Belgrave return "%u";
1277aa3b2b4cSBeau Belgrave if (strcmp(type, "int") == 0)
1278aa3b2b4cSBeau Belgrave return "%d";
1279aa3b2b4cSBeau Belgrave if (strcmp(type, "unsigned int") == 0)
1280aa3b2b4cSBeau Belgrave return "%u";
1281aa3b2b4cSBeau Belgrave if (strcmp(type, "s16") == 0)
1282aa3b2b4cSBeau Belgrave return "%d";
1283aa3b2b4cSBeau Belgrave if (strcmp(type, "u16") == 0)
1284aa3b2b4cSBeau Belgrave return "%u";
1285aa3b2b4cSBeau Belgrave if (strcmp(type, "short") == 0)
1286aa3b2b4cSBeau Belgrave return "%d";
1287aa3b2b4cSBeau Belgrave if (strcmp(type, "unsigned short") == 0)
1288aa3b2b4cSBeau Belgrave return "%u";
1289aa3b2b4cSBeau Belgrave if (strcmp(type, "s8") == 0)
1290aa3b2b4cSBeau Belgrave return "%d";
1291aa3b2b4cSBeau Belgrave if (strcmp(type, "u8") == 0)
1292aa3b2b4cSBeau Belgrave return "%u";
1293aa3b2b4cSBeau Belgrave if (strcmp(type, "char") == 0)
1294aa3b2b4cSBeau Belgrave return "%d";
1295aa3b2b4cSBeau Belgrave if (strcmp(type, "unsigned char") == 0)
1296aa3b2b4cSBeau Belgrave return "%u";
12979cbf1234SBeau Belgrave if (strstr(type, "char[") != NULL)
1298aa3b2b4cSBeau Belgrave return "%s";
1299aa3b2b4cSBeau Belgrave
1300aa3b2b4cSBeau Belgrave /* Unknown, likely struct, allowed treat as 64-bit */
1301aa3b2b4cSBeau Belgrave return "%llu";
1302aa3b2b4cSBeau Belgrave }
1303aa3b2b4cSBeau Belgrave
user_field_is_dyn_string(const char * type,const char ** str_func)1304aa3b2b4cSBeau Belgrave static bool user_field_is_dyn_string(const char *type, const char **str_func)
1305aa3b2b4cSBeau Belgrave {
1306aa3b2b4cSBeau Belgrave if (str_has_prefix(type, "__data_loc ")) {
1307aa3b2b4cSBeau Belgrave *str_func = "__get_str";
1308aa3b2b4cSBeau Belgrave goto check;
1309aa3b2b4cSBeau Belgrave }
1310aa3b2b4cSBeau Belgrave
1311aa3b2b4cSBeau Belgrave if (str_has_prefix(type, "__rel_loc ")) {
1312aa3b2b4cSBeau Belgrave *str_func = "__get_rel_str";
1313aa3b2b4cSBeau Belgrave goto check;
1314aa3b2b4cSBeau Belgrave }
1315aa3b2b4cSBeau Belgrave
1316aa3b2b4cSBeau Belgrave return false;
1317aa3b2b4cSBeau Belgrave check:
13189cbf1234SBeau Belgrave return strstr(type, "char") != NULL;
1319aa3b2b4cSBeau Belgrave }
1320aa3b2b4cSBeau Belgrave
1321aa3b2b4cSBeau Belgrave #define LEN_OR_ZERO (len ? len - pos : 0)
user_dyn_field_set_string(int argc,const char ** argv,int * iout,char * buf,int len,bool * colon)1322e6f89a14SBeau Belgrave static int user_dyn_field_set_string(int argc, const char **argv, int *iout,
1323e6f89a14SBeau Belgrave char *buf, int len, bool *colon)
1324e6f89a14SBeau Belgrave {
1325e6f89a14SBeau Belgrave int pos = 0, i = *iout;
1326e6f89a14SBeau Belgrave
1327e6f89a14SBeau Belgrave *colon = false;
1328e6f89a14SBeau Belgrave
1329e6f89a14SBeau Belgrave for (; i < argc; ++i) {
1330e6f89a14SBeau Belgrave if (i != *iout)
1331e6f89a14SBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
1332e6f89a14SBeau Belgrave
1333e6f89a14SBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", argv[i]);
1334e6f89a14SBeau Belgrave
1335e6f89a14SBeau Belgrave if (strchr(argv[i], ';')) {
1336e6f89a14SBeau Belgrave ++i;
1337e6f89a14SBeau Belgrave *colon = true;
1338e6f89a14SBeau Belgrave break;
1339e6f89a14SBeau Belgrave }
1340e6f89a14SBeau Belgrave }
1341e6f89a14SBeau Belgrave
1342e6f89a14SBeau Belgrave /* Actual set, advance i */
1343e6f89a14SBeau Belgrave if (len != 0)
1344e6f89a14SBeau Belgrave *iout = i;
1345e6f89a14SBeau Belgrave
1346e6f89a14SBeau Belgrave return pos + 1;
1347e6f89a14SBeau Belgrave }
1348e6f89a14SBeau Belgrave
user_field_set_string(struct ftrace_event_field * field,char * buf,int len,bool colon)1349e6f89a14SBeau Belgrave static int user_field_set_string(struct ftrace_event_field *field,
1350e6f89a14SBeau Belgrave char *buf, int len, bool colon)
1351e6f89a14SBeau Belgrave {
1352e6f89a14SBeau Belgrave int pos = 0;
1353e6f89a14SBeau Belgrave
1354e6f89a14SBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->type);
1355e6f89a14SBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
1356e6f89a14SBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->name);
1357e6f89a14SBeau Belgrave
1358d0a3022fSBeau Belgrave if (str_has_prefix(field->type, "struct "))
1359d0a3022fSBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO, " %d", field->size);
1360d0a3022fSBeau Belgrave
1361e6f89a14SBeau Belgrave if (colon)
1362e6f89a14SBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO, ";");
1363e6f89a14SBeau Belgrave
1364e6f89a14SBeau Belgrave return pos + 1;
1365e6f89a14SBeau Belgrave }
1366e6f89a14SBeau Belgrave
user_event_set_print_fmt(struct user_event * user,char * buf,int len)1367aa3b2b4cSBeau Belgrave static int user_event_set_print_fmt(struct user_event *user, char *buf, int len)
1368aa3b2b4cSBeau Belgrave {
1369a943188dSEric Vaughn struct ftrace_event_field *field;
1370aa3b2b4cSBeau Belgrave struct list_head *head = &user->fields;
1371aa3b2b4cSBeau Belgrave int pos = 0, depth = 0;
1372aa3b2b4cSBeau Belgrave const char *str_func;
1373aa3b2b4cSBeau Belgrave
1374aa3b2b4cSBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
1375aa3b2b4cSBeau Belgrave
1376a943188dSEric Vaughn list_for_each_entry_reverse(field, head, link) {
1377aa3b2b4cSBeau Belgrave if (depth != 0)
1378aa3b2b4cSBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
1379aa3b2b4cSBeau Belgrave
1380aa3b2b4cSBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s",
1381aa3b2b4cSBeau Belgrave field->name, user_field_format(field->type));
1382aa3b2b4cSBeau Belgrave
1383aa3b2b4cSBeau Belgrave depth++;
1384aa3b2b4cSBeau Belgrave }
1385aa3b2b4cSBeau Belgrave
1386aa3b2b4cSBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
1387aa3b2b4cSBeau Belgrave
1388a943188dSEric Vaughn list_for_each_entry_reverse(field, head, link) {
1389aa3b2b4cSBeau Belgrave if (user_field_is_dyn_string(field->type, &str_func))
1390aa3b2b4cSBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO,
1391aa3b2b4cSBeau Belgrave ", %s(%s)", str_func, field->name);
1392aa3b2b4cSBeau Belgrave else
1393aa3b2b4cSBeau Belgrave pos += snprintf(buf + pos, LEN_OR_ZERO,
1394aa3b2b4cSBeau Belgrave ", REC->%s", field->name);
1395aa3b2b4cSBeau Belgrave }
1396aa3b2b4cSBeau Belgrave
1397aa3b2b4cSBeau Belgrave return pos + 1;
1398aa3b2b4cSBeau Belgrave }
1399aa3b2b4cSBeau Belgrave #undef LEN_OR_ZERO
1400aa3b2b4cSBeau Belgrave
user_event_create_print_fmt(struct user_event * user)1401aa3b2b4cSBeau Belgrave static int user_event_create_print_fmt(struct user_event *user)
1402aa3b2b4cSBeau Belgrave {
1403aa3b2b4cSBeau Belgrave char *print_fmt;
1404aa3b2b4cSBeau Belgrave int len;
1405aa3b2b4cSBeau Belgrave
1406aa3b2b4cSBeau Belgrave len = user_event_set_print_fmt(user, NULL, 0);
1407aa3b2b4cSBeau Belgrave
1408f9cce238SBeau Belgrave print_fmt = kmalloc(len, GFP_KERNEL_ACCOUNT);
1409aa3b2b4cSBeau Belgrave
1410aa3b2b4cSBeau Belgrave if (!print_fmt)
1411aa3b2b4cSBeau Belgrave return -ENOMEM;
1412aa3b2b4cSBeau Belgrave
1413aa3b2b4cSBeau Belgrave user_event_set_print_fmt(user, print_fmt, len);
1414aa3b2b4cSBeau Belgrave
1415aa3b2b4cSBeau Belgrave user->call.print_fmt = print_fmt;
1416aa3b2b4cSBeau Belgrave
1417aa3b2b4cSBeau Belgrave return 0;
1418aa3b2b4cSBeau Belgrave }
1419aa3b2b4cSBeau Belgrave
user_event_print_trace(struct trace_iterator * iter,int flags,struct trace_event * event)14207f5a08c7SBeau Belgrave static enum print_line_t user_event_print_trace(struct trace_iterator *iter,
14217f5a08c7SBeau Belgrave int flags,
14227f5a08c7SBeau Belgrave struct trace_event *event)
14237f5a08c7SBeau Belgrave {
14244bec284cSSteven Rostedt (Google) return print_event_fields(iter, event);
14257f5a08c7SBeau Belgrave }
14267f5a08c7SBeau Belgrave
14277f5a08c7SBeau Belgrave static struct trace_event_functions user_event_funcs = {
14287f5a08c7SBeau Belgrave .trace = user_event_print_trace,
14297f5a08c7SBeau Belgrave };
14307f5a08c7SBeau Belgrave
user_event_set_call_visible(struct user_event * user,bool visible)1431089331d4SBeau Belgrave static int user_event_set_call_visible(struct user_event *user, bool visible)
1432089331d4SBeau Belgrave {
1433089331d4SBeau Belgrave int ret;
1434089331d4SBeau Belgrave const struct cred *old_cred;
1435089331d4SBeau Belgrave struct cred *cred;
1436089331d4SBeau Belgrave
1437089331d4SBeau Belgrave cred = prepare_creds();
1438089331d4SBeau Belgrave
1439089331d4SBeau Belgrave if (!cred)
1440089331d4SBeau Belgrave return -ENOMEM;
1441089331d4SBeau Belgrave
1442089331d4SBeau Belgrave /*
1443089331d4SBeau Belgrave * While by default tracefs is locked down, systems can be configured
1444089331d4SBeau Belgrave * to allow user_event files to be less locked down. The extreme case
1445089331d4SBeau Belgrave * being "other" has read/write access to user_events_data/status.
1446089331d4SBeau Belgrave *
144794c255acSXiang wangx * When not locked down, processes may not have permissions to
1448089331d4SBeau Belgrave * add/remove calls themselves to tracefs. We need to temporarily
1449089331d4SBeau Belgrave * switch to root file permission to allow for this scenario.
1450089331d4SBeau Belgrave */
1451089331d4SBeau Belgrave cred->fsuid = GLOBAL_ROOT_UID;
1452089331d4SBeau Belgrave
1453089331d4SBeau Belgrave old_cred = override_creds(cred);
1454089331d4SBeau Belgrave
1455089331d4SBeau Belgrave if (visible)
1456089331d4SBeau Belgrave ret = trace_add_event_call(&user->call);
1457089331d4SBeau Belgrave else
1458089331d4SBeau Belgrave ret = trace_remove_event_call(&user->call);
1459089331d4SBeau Belgrave
1460089331d4SBeau Belgrave revert_creds(old_cred);
1461089331d4SBeau Belgrave put_cred(cred);
1462089331d4SBeau Belgrave
1463089331d4SBeau Belgrave return ret;
1464089331d4SBeau Belgrave }
1465089331d4SBeau Belgrave
destroy_user_event(struct user_event * user)14667f5a08c7SBeau Belgrave static int destroy_user_event(struct user_event *user)
14677f5a08c7SBeau Belgrave {
14687f5a08c7SBeau Belgrave int ret = 0;
14697f5a08c7SBeau Belgrave
1470ce58e96eSBeau Belgrave lockdep_assert_held(&event_mutex);
1471ce58e96eSBeau Belgrave
14727f5a08c7SBeau Belgrave /* Must destroy fields before call removal */
14737f5a08c7SBeau Belgrave user_event_destroy_fields(user);
14747f5a08c7SBeau Belgrave
1475089331d4SBeau Belgrave ret = user_event_set_call_visible(user, false);
14767f5a08c7SBeau Belgrave
14777f5a08c7SBeau Belgrave if (ret)
14787f5a08c7SBeau Belgrave return ret;
14797f5a08c7SBeau Belgrave
14807f5a08c7SBeau Belgrave dyn_event_remove(&user->devent);
14817f5a08c7SBeau Belgrave hash_del(&user->node);
14827f5a08c7SBeau Belgrave
14832467cda1SBeau Belgrave user_event_destroy_validators(user);
1484aa3b2b4cSBeau Belgrave kfree(user->call.print_fmt);
14857f5a08c7SBeau Belgrave kfree(EVENT_NAME(user));
14867f5a08c7SBeau Belgrave kfree(user);
14877f5a08c7SBeau Belgrave
1488ce58e96eSBeau Belgrave if (current_user_events > 0)
1489ce58e96eSBeau Belgrave current_user_events--;
1490ce58e96eSBeau Belgrave else
1491ce58e96eSBeau Belgrave pr_alert("BUG: Bad current_user_events\n");
1492ce58e96eSBeau Belgrave
14937f5a08c7SBeau Belgrave return ret;
14947f5a08c7SBeau Belgrave }
14957f5a08c7SBeau Belgrave
find_user_event(struct user_event_group * group,char * name,u32 * outkey)1496e5d27181SBeau Belgrave static struct user_event *find_user_event(struct user_event_group *group,
1497e5d27181SBeau Belgrave char *name, u32 *outkey)
14987f5a08c7SBeau Belgrave {
14997f5a08c7SBeau Belgrave struct user_event *user;
15007f5a08c7SBeau Belgrave u32 key = user_event_key(name);
15017f5a08c7SBeau Belgrave
15027f5a08c7SBeau Belgrave *outkey = key;
15037f5a08c7SBeau Belgrave
1504e5d27181SBeau Belgrave hash_for_each_possible(group->register_table, user, node, key)
1505f0dbf6fdSBeau Belgrave if (!strcmp(EVENT_NAME(user), name))
1506f0dbf6fdSBeau Belgrave return user_event_get(user);
15077f5a08c7SBeau Belgrave
15087f5a08c7SBeau Belgrave return NULL;
15097f5a08c7SBeau Belgrave }
15107f5a08c7SBeau Belgrave
user_event_validate(struct user_event * user,void * data,int len)15112467cda1SBeau Belgrave static int user_event_validate(struct user_event *user, void *data, int len)
15122467cda1SBeau Belgrave {
15132467cda1SBeau Belgrave struct list_head *head = &user->validators;
15142467cda1SBeau Belgrave struct user_event_validator *validator;
15152467cda1SBeau Belgrave void *pos, *end = data + len;
15162467cda1SBeau Belgrave u32 loc, offset, size;
15172467cda1SBeau Belgrave
1518dcbd1ac2SBeau Belgrave list_for_each_entry(validator, head, user_event_link) {
15192467cda1SBeau Belgrave pos = data + validator->offset;
15202467cda1SBeau Belgrave
15212467cda1SBeau Belgrave /* Already done min_size check, no bounds check here */
15222467cda1SBeau Belgrave loc = *(u32 *)pos;
15232467cda1SBeau Belgrave offset = loc & 0xffff;
15242467cda1SBeau Belgrave size = loc >> 16;
15252467cda1SBeau Belgrave
15262467cda1SBeau Belgrave if (likely(validator->flags & VALIDATOR_REL))
15272467cda1SBeau Belgrave pos += offset + sizeof(loc);
15282467cda1SBeau Belgrave else
15292467cda1SBeau Belgrave pos = data + offset;
15302467cda1SBeau Belgrave
15312467cda1SBeau Belgrave pos += size;
15322467cda1SBeau Belgrave
15332467cda1SBeau Belgrave if (unlikely(pos > end))
15342467cda1SBeau Belgrave return -EFAULT;
15352467cda1SBeau Belgrave
15362467cda1SBeau Belgrave if (likely(validator->flags & VALIDATOR_ENSURE_NULL))
15372467cda1SBeau Belgrave if (unlikely(*(char *)(pos - 1) != '\0'))
15382467cda1SBeau Belgrave return -EFAULT;
15392467cda1SBeau Belgrave }
15402467cda1SBeau Belgrave
15412467cda1SBeau Belgrave return 0;
15422467cda1SBeau Belgrave }
15432467cda1SBeau Belgrave
15447f5a08c7SBeau Belgrave /*
15457f5a08c7SBeau Belgrave * Writes the user supplied payload out to a trace file.
15467f5a08c7SBeau Belgrave */
user_event_ftrace(struct user_event * user,struct iov_iter * i,void * tpdata,bool * faulted)15470279400aSBeau Belgrave static void user_event_ftrace(struct user_event *user, struct iov_iter *i,
15482467cda1SBeau Belgrave void *tpdata, bool *faulted)
15497f5a08c7SBeau Belgrave {
15507f5a08c7SBeau Belgrave struct trace_event_file *file;
15517f5a08c7SBeau Belgrave struct trace_entry *entry;
15527f5a08c7SBeau Belgrave struct trace_event_buffer event_buffer;
15532467cda1SBeau Belgrave size_t size = sizeof(*entry) + i->count;
15547f5a08c7SBeau Belgrave
15557f5a08c7SBeau Belgrave file = (struct trace_event_file *)tpdata;
15567f5a08c7SBeau Belgrave
15577f5a08c7SBeau Belgrave if (!file ||
15587f5a08c7SBeau Belgrave !(file->flags & EVENT_FILE_FL_ENABLED) ||
15597f5a08c7SBeau Belgrave trace_trigger_soft_disabled(file))
15607f5a08c7SBeau Belgrave return;
15617f5a08c7SBeau Belgrave
15627f5a08c7SBeau Belgrave /* Allocates and fills trace_entry, + 1 of this is data payload */
15632467cda1SBeau Belgrave entry = trace_event_buffer_reserve(&event_buffer, file, size);
15647f5a08c7SBeau Belgrave
15657f5a08c7SBeau Belgrave if (unlikely(!entry))
15667f5a08c7SBeau Belgrave return;
15677f5a08c7SBeau Belgrave
15686f05dcabSsunliming if (unlikely(i->count != 0 && !copy_nofault(entry + 1, i->count, i)))
15692467cda1SBeau Belgrave goto discard;
15702467cda1SBeau Belgrave
15712467cda1SBeau Belgrave if (!list_empty(&user->validators) &&
15722467cda1SBeau Belgrave unlikely(user_event_validate(user, entry, size)))
15732467cda1SBeau Belgrave goto discard;
15742467cda1SBeau Belgrave
15752467cda1SBeau Belgrave trace_event_buffer_commit(&event_buffer);
15762467cda1SBeau Belgrave
15772467cda1SBeau Belgrave return;
15782467cda1SBeau Belgrave discard:
15792467cda1SBeau Belgrave *faulted = true;
15800279400aSBeau Belgrave __trace_event_discard_commit(event_buffer.buffer,
15810279400aSBeau Belgrave event_buffer.event);
15827f5a08c7SBeau Belgrave }
15837f5a08c7SBeau Belgrave
15843207d045SBeau Belgrave #ifdef CONFIG_PERF_EVENTS
15853207d045SBeau Belgrave /*
1586768c1e7fSBeau Belgrave * Writes the user supplied payload out to perf ring buffer.
15873207d045SBeau Belgrave */
user_event_perf(struct user_event * user,struct iov_iter * i,void * tpdata,bool * faulted)15880279400aSBeau Belgrave static void user_event_perf(struct user_event *user, struct iov_iter *i,
15892467cda1SBeau Belgrave void *tpdata, bool *faulted)
15903207d045SBeau Belgrave {
15913207d045SBeau Belgrave struct hlist_head *perf_head;
15923207d045SBeau Belgrave
15933207d045SBeau Belgrave perf_head = this_cpu_ptr(user->call.perf_events);
15943207d045SBeau Belgrave
15953207d045SBeau Belgrave if (perf_head && !hlist_empty(perf_head)) {
15963207d045SBeau Belgrave struct trace_entry *perf_entry;
15973207d045SBeau Belgrave struct pt_regs *regs;
15980279400aSBeau Belgrave size_t size = sizeof(*perf_entry) + i->count;
15993207d045SBeau Belgrave int context;
16003207d045SBeau Belgrave
16013207d045SBeau Belgrave perf_entry = perf_trace_buf_alloc(ALIGN(size, 8),
16023207d045SBeau Belgrave ®s, &context);
16033207d045SBeau Belgrave
16043207d045SBeau Belgrave if (unlikely(!perf_entry))
16053207d045SBeau Belgrave return;
16063207d045SBeau Belgrave
16073207d045SBeau Belgrave perf_fetch_caller_regs(regs);
16083207d045SBeau Belgrave
16096f05dcabSsunliming if (unlikely(i->count != 0 && !copy_nofault(perf_entry + 1, i->count, i)))
16102467cda1SBeau Belgrave goto discard;
16112467cda1SBeau Belgrave
16122467cda1SBeau Belgrave if (!list_empty(&user->validators) &&
16132467cda1SBeau Belgrave unlikely(user_event_validate(user, perf_entry, size)))
16142467cda1SBeau Belgrave goto discard;
16153207d045SBeau Belgrave
16163207d045SBeau Belgrave perf_trace_buf_submit(perf_entry, size, context,
16173207d045SBeau Belgrave user->call.event.type, 1, regs,
16183207d045SBeau Belgrave perf_head, NULL);
16192467cda1SBeau Belgrave
16202467cda1SBeau Belgrave return;
16212467cda1SBeau Belgrave discard:
16222467cda1SBeau Belgrave *faulted = true;
16232467cda1SBeau Belgrave perf_swevent_put_recursion_context(context);
16243207d045SBeau Belgrave }
16253207d045SBeau Belgrave }
16263207d045SBeau Belgrave #endif
16273207d045SBeau Belgrave
16287f5a08c7SBeau Belgrave /*
162972357590SBeau Belgrave * Update the enabled bit among all user processes.
16307f5a08c7SBeau Belgrave */
update_enable_bit_for(struct user_event * user)163172357590SBeau Belgrave static void update_enable_bit_for(struct user_event *user)
16327f5a08c7SBeau Belgrave {
16337f5a08c7SBeau Belgrave struct tracepoint *tp = &user->tracepoint;
16347f5a08c7SBeau Belgrave char status = 0;
16357f5a08c7SBeau Belgrave
16367f5a08c7SBeau Belgrave if (atomic_read(&tp->key.enabled) > 0) {
16377f5a08c7SBeau Belgrave struct tracepoint_func *probe_func_ptr;
16387f5a08c7SBeau Belgrave user_event_func_t probe_func;
16397f5a08c7SBeau Belgrave
16407f5a08c7SBeau Belgrave rcu_read_lock_sched();
16417f5a08c7SBeau Belgrave
16427f5a08c7SBeau Belgrave probe_func_ptr = rcu_dereference_sched(tp->funcs);
16437f5a08c7SBeau Belgrave
16447f5a08c7SBeau Belgrave if (probe_func_ptr) {
16457f5a08c7SBeau Belgrave do {
16467f5a08c7SBeau Belgrave probe_func = probe_func_ptr->func;
16477f5a08c7SBeau Belgrave
16487f5a08c7SBeau Belgrave if (probe_func == user_event_ftrace)
16497f5a08c7SBeau Belgrave status |= EVENT_STATUS_FTRACE;
16503207d045SBeau Belgrave #ifdef CONFIG_PERF_EVENTS
16513207d045SBeau Belgrave else if (probe_func == user_event_perf)
16523207d045SBeau Belgrave status |= EVENT_STATUS_PERF;
16533207d045SBeau Belgrave #endif
16547f5a08c7SBeau Belgrave else
16557f5a08c7SBeau Belgrave status |= EVENT_STATUS_OTHER;
16567f5a08c7SBeau Belgrave } while ((++probe_func_ptr)->func);
16577f5a08c7SBeau Belgrave }
16587f5a08c7SBeau Belgrave
16597f5a08c7SBeau Belgrave rcu_read_unlock_sched();
16607f5a08c7SBeau Belgrave }
16617f5a08c7SBeau Belgrave
166239d6d08bSBeau Belgrave user->status = status;
166372357590SBeau Belgrave
166472357590SBeau Belgrave user_event_enabler_update(user);
16657f5a08c7SBeau Belgrave }
16667f5a08c7SBeau Belgrave
16677f5a08c7SBeau Belgrave /*
16687f5a08c7SBeau Belgrave * Register callback for our events from tracing sub-systems.
16697f5a08c7SBeau Belgrave */
user_event_reg(struct trace_event_call * call,enum trace_reg type,void * data)16707f5a08c7SBeau Belgrave static int user_event_reg(struct trace_event_call *call,
16717f5a08c7SBeau Belgrave enum trace_reg type,
16727f5a08c7SBeau Belgrave void *data)
16737f5a08c7SBeau Belgrave {
16747f5a08c7SBeau Belgrave struct user_event *user = (struct user_event *)call->data;
16757f5a08c7SBeau Belgrave int ret = 0;
16767f5a08c7SBeau Belgrave
16777f5a08c7SBeau Belgrave if (!user)
16787f5a08c7SBeau Belgrave return -ENOENT;
16797f5a08c7SBeau Belgrave
16807f5a08c7SBeau Belgrave switch (type) {
16817f5a08c7SBeau Belgrave case TRACE_REG_REGISTER:
16827f5a08c7SBeau Belgrave ret = tracepoint_probe_register(call->tp,
16837f5a08c7SBeau Belgrave call->class->probe,
16847f5a08c7SBeau Belgrave data);
16857f5a08c7SBeau Belgrave if (!ret)
16867f5a08c7SBeau Belgrave goto inc;
16877f5a08c7SBeau Belgrave break;
16887f5a08c7SBeau Belgrave
16897f5a08c7SBeau Belgrave case TRACE_REG_UNREGISTER:
16907f5a08c7SBeau Belgrave tracepoint_probe_unregister(call->tp,
16917f5a08c7SBeau Belgrave call->class->probe,
16927f5a08c7SBeau Belgrave data);
16937f5a08c7SBeau Belgrave goto dec;
16947f5a08c7SBeau Belgrave
16953207d045SBeau Belgrave #ifdef CONFIG_PERF_EVENTS
16963207d045SBeau Belgrave case TRACE_REG_PERF_REGISTER:
16973207d045SBeau Belgrave ret = tracepoint_probe_register(call->tp,
16983207d045SBeau Belgrave call->class->perf_probe,
16993207d045SBeau Belgrave data);
17003207d045SBeau Belgrave if (!ret)
17013207d045SBeau Belgrave goto inc;
17027f5a08c7SBeau Belgrave break;
17033207d045SBeau Belgrave
17043207d045SBeau Belgrave case TRACE_REG_PERF_UNREGISTER:
17053207d045SBeau Belgrave tracepoint_probe_unregister(call->tp,
17063207d045SBeau Belgrave call->class->perf_probe,
17073207d045SBeau Belgrave data);
17083207d045SBeau Belgrave goto dec;
17093207d045SBeau Belgrave
17103207d045SBeau Belgrave case TRACE_REG_PERF_OPEN:
17113207d045SBeau Belgrave case TRACE_REG_PERF_CLOSE:
17123207d045SBeau Belgrave case TRACE_REG_PERF_ADD:
17133207d045SBeau Belgrave case TRACE_REG_PERF_DEL:
17143207d045SBeau Belgrave break;
17153207d045SBeau Belgrave #endif
17167f5a08c7SBeau Belgrave }
17177f5a08c7SBeau Belgrave
17187f5a08c7SBeau Belgrave return ret;
17197f5a08c7SBeau Belgrave inc:
1720f0dbf6fdSBeau Belgrave user_event_get(user);
172172357590SBeau Belgrave update_enable_bit_for(user);
17227f5a08c7SBeau Belgrave return 0;
17237f5a08c7SBeau Belgrave dec:
172472357590SBeau Belgrave update_enable_bit_for(user);
1725f0dbf6fdSBeau Belgrave user_event_put(user, true);
17267f5a08c7SBeau Belgrave return 0;
17277f5a08c7SBeau Belgrave }
17287f5a08c7SBeau Belgrave
user_event_create(const char * raw_command)17297f5a08c7SBeau Belgrave static int user_event_create(const char *raw_command)
17307f5a08c7SBeau Belgrave {
1731e5d27181SBeau Belgrave struct user_event_group *group;
17327f5a08c7SBeau Belgrave struct user_event *user;
17337f5a08c7SBeau Belgrave char *name;
17347f5a08c7SBeau Belgrave int ret;
17357f5a08c7SBeau Belgrave
17367f5a08c7SBeau Belgrave if (!str_has_prefix(raw_command, USER_EVENTS_PREFIX))
17377f5a08c7SBeau Belgrave return -ECANCELED;
17387f5a08c7SBeau Belgrave
17397f5a08c7SBeau Belgrave raw_command += USER_EVENTS_PREFIX_LEN;
17407f5a08c7SBeau Belgrave raw_command = skip_spaces(raw_command);
17417f5a08c7SBeau Belgrave
1742f9cce238SBeau Belgrave name = kstrdup(raw_command, GFP_KERNEL_ACCOUNT);
17437f5a08c7SBeau Belgrave
17447f5a08c7SBeau Belgrave if (!name)
17457f5a08c7SBeau Belgrave return -ENOMEM;
17467f5a08c7SBeau Belgrave
1747e5d27181SBeau Belgrave group = current_user_event_group();
17487e348b32SBeau Belgrave
1749ccc6e590SXiu Jianfeng if (!group) {
1750ccc6e590SXiu Jianfeng kfree(name);
1751e5d27181SBeau Belgrave return -ENOENT;
1752ccc6e590SXiu Jianfeng }
1753e5d27181SBeau Belgrave
1754e5d27181SBeau Belgrave mutex_lock(&group->reg_mutex);
1755e5d27181SBeau Belgrave
1756a65442edSBeau Belgrave /* Dyn events persist, otherwise they would cleanup immediately */
1757a65442edSBeau Belgrave ret = user_event_parse_cmd(group, name, &user, USER_EVENT_REG_PERSIST);
17587e348b32SBeau Belgrave
17597e348b32SBeau Belgrave if (!ret)
1760f0dbf6fdSBeau Belgrave user_event_put(user, false);
17617e348b32SBeau Belgrave
1762e5d27181SBeau Belgrave mutex_unlock(&group->reg_mutex);
17637f5a08c7SBeau Belgrave
17647f5a08c7SBeau Belgrave if (ret)
17657f5a08c7SBeau Belgrave kfree(name);
17667f5a08c7SBeau Belgrave
17677f5a08c7SBeau Belgrave return ret;
17687f5a08c7SBeau Belgrave }
17697f5a08c7SBeau Belgrave
user_event_show(struct seq_file * m,struct dyn_event * ev)17707f5a08c7SBeau Belgrave static int user_event_show(struct seq_file *m, struct dyn_event *ev)
17717f5a08c7SBeau Belgrave {
17727f5a08c7SBeau Belgrave struct user_event *user = container_of(ev, struct user_event, devent);
1773a943188dSEric Vaughn struct ftrace_event_field *field;
17747f5a08c7SBeau Belgrave struct list_head *head;
17757f5a08c7SBeau Belgrave int depth = 0;
17767f5a08c7SBeau Belgrave
17777f5a08c7SBeau Belgrave seq_printf(m, "%s%s", USER_EVENTS_PREFIX, EVENT_NAME(user));
17787f5a08c7SBeau Belgrave
17797f5a08c7SBeau Belgrave head = trace_get_fields(&user->call);
17807f5a08c7SBeau Belgrave
1781a943188dSEric Vaughn list_for_each_entry_reverse(field, head, link) {
17827f5a08c7SBeau Belgrave if (depth == 0)
17837f5a08c7SBeau Belgrave seq_puts(m, " ");
17847f5a08c7SBeau Belgrave else
17857f5a08c7SBeau Belgrave seq_puts(m, "; ");
17867f5a08c7SBeau Belgrave
17877f5a08c7SBeau Belgrave seq_printf(m, "%s %s", field->type, field->name);
17887f5a08c7SBeau Belgrave
17897f5a08c7SBeau Belgrave if (str_has_prefix(field->type, "struct "))
17907f5a08c7SBeau Belgrave seq_printf(m, " %d", field->size);
17917f5a08c7SBeau Belgrave
17927f5a08c7SBeau Belgrave depth++;
17937f5a08c7SBeau Belgrave }
17947f5a08c7SBeau Belgrave
17957f5a08c7SBeau Belgrave seq_puts(m, "\n");
17967f5a08c7SBeau Belgrave
17977f5a08c7SBeau Belgrave return 0;
17987f5a08c7SBeau Belgrave }
17997f5a08c7SBeau Belgrave
user_event_is_busy(struct dyn_event * ev)18007f5a08c7SBeau Belgrave static bool user_event_is_busy(struct dyn_event *ev)
18017f5a08c7SBeau Belgrave {
18027f5a08c7SBeau Belgrave struct user_event *user = container_of(ev, struct user_event, devent);
18037f5a08c7SBeau Belgrave
1804d401b724SBeau Belgrave return !user_event_last_ref(user);
18057f5a08c7SBeau Belgrave }
18067f5a08c7SBeau Belgrave
user_event_free(struct dyn_event * ev)18077f5a08c7SBeau Belgrave static int user_event_free(struct dyn_event *ev)
18087f5a08c7SBeau Belgrave {
18097f5a08c7SBeau Belgrave struct user_event *user = container_of(ev, struct user_event, devent);
18107f5a08c7SBeau Belgrave
1811d401b724SBeau Belgrave if (!user_event_last_ref(user))
18127f5a08c7SBeau Belgrave return -EBUSY;
18137f5a08c7SBeau Belgrave
18147f5a08c7SBeau Belgrave return destroy_user_event(user);
18157f5a08c7SBeau Belgrave }
18167f5a08c7SBeau Belgrave
user_field_match(struct ftrace_event_field * field,int argc,const char ** argv,int * iout)18179aed4e15SBeau Belgrave static bool user_field_match(struct ftrace_event_field *field, int argc,
18189aed4e15SBeau Belgrave const char **argv, int *iout)
18199aed4e15SBeau Belgrave {
1820e6f89a14SBeau Belgrave char *field_name = NULL, *dyn_field_name = NULL;
18219aed4e15SBeau Belgrave bool colon = false, match = false;
1822e6f89a14SBeau Belgrave int dyn_len, len;
18239aed4e15SBeau Belgrave
1824e6f89a14SBeau Belgrave if (*iout >= argc)
18259aed4e15SBeau Belgrave return false;
18269aed4e15SBeau Belgrave
1827e6f89a14SBeau Belgrave dyn_len = user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
1828e6f89a14SBeau Belgrave 0, &colon);
18299aed4e15SBeau Belgrave
1830e6f89a14SBeau Belgrave len = user_field_set_string(field, field_name, 0, colon);
1831e6f89a14SBeau Belgrave
1832e6f89a14SBeau Belgrave if (dyn_len != len)
1833e6f89a14SBeau Belgrave return false;
1834e6f89a14SBeau Belgrave
1835e6f89a14SBeau Belgrave dyn_field_name = kmalloc(dyn_len, GFP_KERNEL);
1836e6f89a14SBeau Belgrave field_name = kmalloc(len, GFP_KERNEL);
1837e6f89a14SBeau Belgrave
1838e6f89a14SBeau Belgrave if (!dyn_field_name || !field_name)
18399aed4e15SBeau Belgrave goto out;
18409aed4e15SBeau Belgrave
1841e6f89a14SBeau Belgrave user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
1842e6f89a14SBeau Belgrave dyn_len, &colon);
18439aed4e15SBeau Belgrave
1844e6f89a14SBeau Belgrave user_field_set_string(field, field_name, len, colon);
18459aed4e15SBeau Belgrave
1846e6f89a14SBeau Belgrave match = strcmp(dyn_field_name, field_name) == 0;
18479aed4e15SBeau Belgrave out:
1848e6f89a14SBeau Belgrave kfree(dyn_field_name);
18499aed4e15SBeau Belgrave kfree(field_name);
18509aed4e15SBeau Belgrave
18519aed4e15SBeau Belgrave return match;
18529aed4e15SBeau Belgrave }
18539aed4e15SBeau Belgrave
user_fields_match(struct user_event * user,int argc,const char ** argv)18549aed4e15SBeau Belgrave static bool user_fields_match(struct user_event *user, int argc,
18559aed4e15SBeau Belgrave const char **argv)
18569aed4e15SBeau Belgrave {
1857a943188dSEric Vaughn struct ftrace_event_field *field;
18589aed4e15SBeau Belgrave struct list_head *head = &user->fields;
18599aed4e15SBeau Belgrave int i = 0;
18609aed4e15SBeau Belgrave
1861a943188dSEric Vaughn list_for_each_entry_reverse(field, head, link) {
18629aed4e15SBeau Belgrave if (!user_field_match(field, argc, argv, &i))
18639aed4e15SBeau Belgrave return false;
1864a943188dSEric Vaughn }
18659aed4e15SBeau Belgrave
18669aed4e15SBeau Belgrave if (i != argc)
18679aed4e15SBeau Belgrave return false;
18689aed4e15SBeau Belgrave
18699aed4e15SBeau Belgrave return true;
18709aed4e15SBeau Belgrave }
18719aed4e15SBeau Belgrave
user_event_match(const char * system,const char * event,int argc,const char ** argv,struct dyn_event * ev)18727f5a08c7SBeau Belgrave static bool user_event_match(const char *system, const char *event,
18737f5a08c7SBeau Belgrave int argc, const char **argv, struct dyn_event *ev)
18747f5a08c7SBeau Belgrave {
18757f5a08c7SBeau Belgrave struct user_event *user = container_of(ev, struct user_event, devent);
18769aed4e15SBeau Belgrave bool match;
18777f5a08c7SBeau Belgrave
18789aed4e15SBeau Belgrave match = strcmp(EVENT_NAME(user), event) == 0 &&
18797f5a08c7SBeau Belgrave (!system || strcmp(system, USER_EVENTS_SYSTEM) == 0);
18809aed4e15SBeau Belgrave
18819aed4e15SBeau Belgrave if (match && argc > 0)
18829aed4e15SBeau Belgrave match = user_fields_match(user, argc, argv);
1883cfac4ed7Ssunliming else if (match && argc == 0)
1884cfac4ed7Ssunliming match = list_empty(&user->fields);
18859aed4e15SBeau Belgrave
18869aed4e15SBeau Belgrave return match;
18877f5a08c7SBeau Belgrave }
18887f5a08c7SBeau Belgrave
18897f5a08c7SBeau Belgrave static struct dyn_event_operations user_event_dops = {
18907f5a08c7SBeau Belgrave .create = user_event_create,
18917f5a08c7SBeau Belgrave .show = user_event_show,
18927f5a08c7SBeau Belgrave .is_busy = user_event_is_busy,
18937f5a08c7SBeau Belgrave .free = user_event_free,
18947f5a08c7SBeau Belgrave .match = user_event_match,
18957f5a08c7SBeau Belgrave };
18967f5a08c7SBeau Belgrave
user_event_trace_register(struct user_event * user)18977f5a08c7SBeau Belgrave static int user_event_trace_register(struct user_event *user)
18987f5a08c7SBeau Belgrave {
18997f5a08c7SBeau Belgrave int ret;
19007f5a08c7SBeau Belgrave
19017f5a08c7SBeau Belgrave ret = register_trace_event(&user->call.event);
19027f5a08c7SBeau Belgrave
19037f5a08c7SBeau Belgrave if (!ret)
19047f5a08c7SBeau Belgrave return -ENODEV;
19057f5a08c7SBeau Belgrave
1906089331d4SBeau Belgrave ret = user_event_set_call_visible(user, true);
19077f5a08c7SBeau Belgrave
19087f5a08c7SBeau Belgrave if (ret)
19097f5a08c7SBeau Belgrave unregister_trace_event(&user->call.event);
19107f5a08c7SBeau Belgrave
19117f5a08c7SBeau Belgrave return ret;
19127f5a08c7SBeau Belgrave }
19137f5a08c7SBeau Belgrave
19147f5a08c7SBeau Belgrave /*
19157f5a08c7SBeau Belgrave * Parses the event name, arguments and flags then registers if successful.
19167f5a08c7SBeau Belgrave * The name buffer lifetime is owned by this method for success cases only.
19177e348b32SBeau Belgrave * Upon success the returned user_event has its ref count increased by 1.
19187f5a08c7SBeau Belgrave */
user_event_parse(struct user_event_group * group,char * name,char * args,char * flags,struct user_event ** newuser,int reg_flags)1919e5d27181SBeau Belgrave static int user_event_parse(struct user_event_group *group, char *name,
1920e5d27181SBeau Belgrave char *args, char *flags,
1921b08d7258SBeau Belgrave struct user_event **newuser, int reg_flags)
19227f5a08c7SBeau Belgrave {
19237f5a08c7SBeau Belgrave int ret;
19247f5a08c7SBeau Belgrave u32 key;
19257e348b32SBeau Belgrave struct user_event *user;
1926ba470eebSsunliming int argc = 0;
1927ba470eebSsunliming char **argv;
19287e348b32SBeau Belgrave
1929a65442edSBeau Belgrave /* User register flags are not ready yet */
1930a65442edSBeau Belgrave if (reg_flags != 0 || flags != NULL)
1931a65442edSBeau Belgrave return -EINVAL;
1932a65442edSBeau Belgrave
19337e348b32SBeau Belgrave /* Prevent dyn_event from racing */
19347e348b32SBeau Belgrave mutex_lock(&event_mutex);
1935e5d27181SBeau Belgrave user = find_user_event(group, name, &key);
19367e348b32SBeau Belgrave mutex_unlock(&event_mutex);
19377f5a08c7SBeau Belgrave
19387f5a08c7SBeau Belgrave if (user) {
1939ba470eebSsunliming if (args) {
1940ba470eebSsunliming argv = argv_split(GFP_KERNEL, args, &argc);
1941ba470eebSsunliming if (!argv) {
1942ba470eebSsunliming ret = -ENOMEM;
1943ba470eebSsunliming goto error;
1944ba470eebSsunliming }
1945ba470eebSsunliming
1946ba470eebSsunliming ret = user_fields_match(user, argc, (const char **)argv);
1947ba470eebSsunliming argv_free(argv);
1948ba470eebSsunliming
1949ba470eebSsunliming } else
1950ba470eebSsunliming ret = list_empty(&user->fields);
1951ba470eebSsunliming
1952ba470eebSsunliming if (ret) {
19537f5a08c7SBeau Belgrave *newuser = user;
19547f5a08c7SBeau Belgrave /*
19557f5a08c7SBeau Belgrave * Name is allocated by caller, free it since it already exists.
19567f5a08c7SBeau Belgrave * Caller only worries about failure cases for freeing.
19577f5a08c7SBeau Belgrave */
19587f5a08c7SBeau Belgrave kfree(name);
1959ba470eebSsunliming } else {
1960ba470eebSsunliming ret = -EADDRINUSE;
1961ba470eebSsunliming goto error;
1962ba470eebSsunliming }
1963ba470eebSsunliming
19647f5a08c7SBeau Belgrave return 0;
1965ba470eebSsunliming error:
1966f0dbf6fdSBeau Belgrave user_event_put(user, false);
1967ba470eebSsunliming return ret;
19687f5a08c7SBeau Belgrave }
19697f5a08c7SBeau Belgrave
1970f9cce238SBeau Belgrave user = kzalloc(sizeof(*user), GFP_KERNEL_ACCOUNT);
19717f5a08c7SBeau Belgrave
19727f5a08c7SBeau Belgrave if (!user)
19737f5a08c7SBeau Belgrave return -ENOMEM;
19747f5a08c7SBeau Belgrave
19757f5a08c7SBeau Belgrave INIT_LIST_HEAD(&user->class.fields);
19767f5a08c7SBeau Belgrave INIT_LIST_HEAD(&user->fields);
19772467cda1SBeau Belgrave INIT_LIST_HEAD(&user->validators);
19787f5a08c7SBeau Belgrave
1979e5d27181SBeau Belgrave user->group = group;
19807f5a08c7SBeau Belgrave user->tracepoint.name = name;
19817f5a08c7SBeau Belgrave
19827f5a08c7SBeau Belgrave ret = user_event_parse_fields(user, args);
19837f5a08c7SBeau Belgrave
19847f5a08c7SBeau Belgrave if (ret)
19857f5a08c7SBeau Belgrave goto put_user;
19867f5a08c7SBeau Belgrave
1987aa3b2b4cSBeau Belgrave ret = user_event_create_print_fmt(user);
1988aa3b2b4cSBeau Belgrave
1989aa3b2b4cSBeau Belgrave if (ret)
1990aa3b2b4cSBeau Belgrave goto put_user;
19917f5a08c7SBeau Belgrave
19927f5a08c7SBeau Belgrave user->call.data = user;
19937f5a08c7SBeau Belgrave user->call.class = &user->class;
19947f5a08c7SBeau Belgrave user->call.name = name;
19957f5a08c7SBeau Belgrave user->call.flags = TRACE_EVENT_FL_TRACEPOINT;
19967f5a08c7SBeau Belgrave user->call.tp = &user->tracepoint;
19977f5a08c7SBeau Belgrave user->call.event.funcs = &user_event_funcs;
1998e5d27181SBeau Belgrave user->class.system = group->system_name;
19997f5a08c7SBeau Belgrave
20007f5a08c7SBeau Belgrave user->class.fields_array = user_event_fields_array;
20017f5a08c7SBeau Belgrave user->class.get_fields = user_event_get_fields;
20027f5a08c7SBeau Belgrave user->class.reg = user_event_reg;
20037f5a08c7SBeau Belgrave user->class.probe = user_event_ftrace;
20043207d045SBeau Belgrave #ifdef CONFIG_PERF_EVENTS
20053207d045SBeau Belgrave user->class.perf_probe = user_event_perf;
20063207d045SBeau Belgrave #endif
20077f5a08c7SBeau Belgrave
20087f5a08c7SBeau Belgrave mutex_lock(&event_mutex);
2009efe34e99SBeau Belgrave
2010ce58e96eSBeau Belgrave if (current_user_events >= max_user_events) {
2011ce58e96eSBeau Belgrave ret = -EMFILE;
2012ce58e96eSBeau Belgrave goto put_user_lock;
2013ce58e96eSBeau Belgrave }
2014ce58e96eSBeau Belgrave
20157f5a08c7SBeau Belgrave ret = user_event_trace_register(user);
20167f5a08c7SBeau Belgrave
20177f5a08c7SBeau Belgrave if (ret)
2018efe34e99SBeau Belgrave goto put_user_lock;
20197f5a08c7SBeau Belgrave
2020b08d7258SBeau Belgrave user->reg_flags = reg_flags;
2021b08d7258SBeau Belgrave
2022a65442edSBeau Belgrave if (user->reg_flags & USER_EVENT_REG_PERSIST) {
2023d401b724SBeau Belgrave /* Ensure we track self ref and caller ref (2) */
2024d401b724SBeau Belgrave refcount_set(&user->refcnt, 2);
2025a65442edSBeau Belgrave } else {
2026a65442edSBeau Belgrave /* Ensure we track only caller ref (1) */
2027a65442edSBeau Belgrave refcount_set(&user->refcnt, 1);
2028a65442edSBeau Belgrave }
20297e348b32SBeau Belgrave
20307f5a08c7SBeau Belgrave dyn_event_init(&user->devent, &user_event_dops);
20317f5a08c7SBeau Belgrave dyn_event_add(&user->devent, &user->call);
2032e5d27181SBeau Belgrave hash_add(group->register_table, &user->node, key);
2033ce58e96eSBeau Belgrave current_user_events++;
20347f5a08c7SBeau Belgrave
2035efe34e99SBeau Belgrave mutex_unlock(&event_mutex);
2036efe34e99SBeau Belgrave
20377f5a08c7SBeau Belgrave *newuser = user;
20387f5a08c7SBeau Belgrave return 0;
2039efe34e99SBeau Belgrave put_user_lock:
2040efe34e99SBeau Belgrave mutex_unlock(&event_mutex);
20417f5a08c7SBeau Belgrave put_user:
20427f5a08c7SBeau Belgrave user_event_destroy_fields(user);
20432467cda1SBeau Belgrave user_event_destroy_validators(user);
20444bded7afSBeau Belgrave kfree(user->call.print_fmt);
20457f5a08c7SBeau Belgrave kfree(user);
20467f5a08c7SBeau Belgrave return ret;
20477f5a08c7SBeau Belgrave }
20487f5a08c7SBeau Belgrave
20497f5a08c7SBeau Belgrave /*
20507f5a08c7SBeau Belgrave * Deletes a previously created event if it is no longer being used.
20517f5a08c7SBeau Belgrave */
delete_user_event(struct user_event_group * group,char * name)2052e5d27181SBeau Belgrave static int delete_user_event(struct user_event_group *group, char *name)
20537f5a08c7SBeau Belgrave {
20547f5a08c7SBeau Belgrave u32 key;
2055e5d27181SBeau Belgrave struct user_event *user = find_user_event(group, name, &key);
20567f5a08c7SBeau Belgrave
20577f5a08c7SBeau Belgrave if (!user)
20587f5a08c7SBeau Belgrave return -ENOENT;
20597f5a08c7SBeau Belgrave
2060f0dbf6fdSBeau Belgrave user_event_put(user, true);
20617f5a08c7SBeau Belgrave
2062d401b724SBeau Belgrave if (!user_event_last_ref(user))
2063d401b724SBeau Belgrave return -EBUSY;
20647e348b32SBeau Belgrave
2065d401b724SBeau Belgrave return destroy_user_event(user);
20667f5a08c7SBeau Belgrave }
20677f5a08c7SBeau Belgrave
20687f5a08c7SBeau Belgrave /*
20697f5a08c7SBeau Belgrave * Validates the user payload and writes via iterator.
20707f5a08c7SBeau Belgrave */
user_events_write_core(struct file * file,struct iov_iter * i)20717f5a08c7SBeau Belgrave static ssize_t user_events_write_core(struct file *file, struct iov_iter *i)
20727f5a08c7SBeau Belgrave {
2073e5d27181SBeau Belgrave struct user_event_file_info *info = file->private_data;
20747f5a08c7SBeau Belgrave struct user_event_refs *refs;
20757f5a08c7SBeau Belgrave struct user_event *user = NULL;
20767f5a08c7SBeau Belgrave struct tracepoint *tp;
20777f5a08c7SBeau Belgrave ssize_t ret = i->count;
20787f5a08c7SBeau Belgrave int idx;
20797f5a08c7SBeau Belgrave
20807f5a08c7SBeau Belgrave if (unlikely(copy_from_iter(&idx, sizeof(idx), i) != sizeof(idx)))
20817f5a08c7SBeau Belgrave return -EFAULT;
20827f5a08c7SBeau Belgrave
2083cd98c932SBeau Belgrave if (idx < 0)
2084cd98c932SBeau Belgrave return -EINVAL;
2085cd98c932SBeau Belgrave
20867f5a08c7SBeau Belgrave rcu_read_lock_sched();
20877f5a08c7SBeau Belgrave
2088e5d27181SBeau Belgrave refs = rcu_dereference_sched(info->refs);
20897f5a08c7SBeau Belgrave
20907f5a08c7SBeau Belgrave /*
20917f5a08c7SBeau Belgrave * The refs->events array is protected by RCU, and new items may be
20927f5a08c7SBeau Belgrave * added. But the user retrieved from indexing into the events array
20937f5a08c7SBeau Belgrave * shall be immutable while the file is opened.
20947f5a08c7SBeau Belgrave */
20957f5a08c7SBeau Belgrave if (likely(refs && idx < refs->count))
20967f5a08c7SBeau Belgrave user = refs->events[idx];
20977f5a08c7SBeau Belgrave
20987f5a08c7SBeau Belgrave rcu_read_unlock_sched();
20997f5a08c7SBeau Belgrave
21007f5a08c7SBeau Belgrave if (unlikely(user == NULL))
21017f5a08c7SBeau Belgrave return -ENOENT;
21027f5a08c7SBeau Belgrave
21032467cda1SBeau Belgrave if (unlikely(i->count < user->min_size))
21042467cda1SBeau Belgrave return -EINVAL;
21052467cda1SBeau Belgrave
21067f5a08c7SBeau Belgrave tp = &user->tracepoint;
21077f5a08c7SBeau Belgrave
21087f5a08c7SBeau Belgrave /*
21097f5a08c7SBeau Belgrave * It's possible key.enabled disables after this check, however
21107f5a08c7SBeau Belgrave * we don't mind if a few events are included in this condition.
21117f5a08c7SBeau Belgrave */
21127f5a08c7SBeau Belgrave if (likely(atomic_read(&tp->key.enabled) > 0)) {
21137f5a08c7SBeau Belgrave struct tracepoint_func *probe_func_ptr;
21147f5a08c7SBeau Belgrave user_event_func_t probe_func;
21150279400aSBeau Belgrave struct iov_iter copy;
21167f5a08c7SBeau Belgrave void *tpdata;
21172467cda1SBeau Belgrave bool faulted;
21187f5a08c7SBeau Belgrave
21190279400aSBeau Belgrave if (unlikely(fault_in_iov_iter_readable(i, i->count)))
21200279400aSBeau Belgrave return -EFAULT;
21217f5a08c7SBeau Belgrave
21222467cda1SBeau Belgrave faulted = false;
21232467cda1SBeau Belgrave
21247f5a08c7SBeau Belgrave rcu_read_lock_sched();
21257f5a08c7SBeau Belgrave
21267f5a08c7SBeau Belgrave probe_func_ptr = rcu_dereference_sched(tp->funcs);
21277f5a08c7SBeau Belgrave
21287f5a08c7SBeau Belgrave if (probe_func_ptr) {
21297f5a08c7SBeau Belgrave do {
21300279400aSBeau Belgrave copy = *i;
21317f5a08c7SBeau Belgrave probe_func = probe_func_ptr->func;
21327f5a08c7SBeau Belgrave tpdata = probe_func_ptr->data;
21332467cda1SBeau Belgrave probe_func(user, ©, tpdata, &faulted);
21347f5a08c7SBeau Belgrave } while ((++probe_func_ptr)->func);
21357f5a08c7SBeau Belgrave }
21367f5a08c7SBeau Belgrave
21377f5a08c7SBeau Belgrave rcu_read_unlock_sched();
21382467cda1SBeau Belgrave
21392467cda1SBeau Belgrave if (unlikely(faulted))
21402467cda1SBeau Belgrave return -EFAULT;
2141f6d026eeSsunliming } else
2142f6d026eeSsunliming return -EBADF;
21437f5a08c7SBeau Belgrave
21447f5a08c7SBeau Belgrave return ret;
21457f5a08c7SBeau Belgrave }
21467f5a08c7SBeau Belgrave
user_events_open(struct inode * node,struct file * file)2147e5d27181SBeau Belgrave static int user_events_open(struct inode *node, struct file *file)
2148e5d27181SBeau Belgrave {
2149e5d27181SBeau Belgrave struct user_event_group *group;
2150e5d27181SBeau Belgrave struct user_event_file_info *info;
2151e5d27181SBeau Belgrave
2152e5d27181SBeau Belgrave group = current_user_event_group();
2153e5d27181SBeau Belgrave
2154e5d27181SBeau Belgrave if (!group)
2155e5d27181SBeau Belgrave return -ENOENT;
2156e5d27181SBeau Belgrave
2157f9cce238SBeau Belgrave info = kzalloc(sizeof(*info), GFP_KERNEL_ACCOUNT);
2158e5d27181SBeau Belgrave
2159e5d27181SBeau Belgrave if (!info)
2160e5d27181SBeau Belgrave return -ENOMEM;
2161e5d27181SBeau Belgrave
2162e5d27181SBeau Belgrave info->group = group;
2163e5d27181SBeau Belgrave
2164e5d27181SBeau Belgrave file->private_data = info;
2165e5d27181SBeau Belgrave
2166e5d27181SBeau Belgrave return 0;
2167e5d27181SBeau Belgrave }
2168e5d27181SBeau Belgrave
user_events_write(struct file * file,const char __user * ubuf,size_t count,loff_t * ppos)21697f5a08c7SBeau Belgrave static ssize_t user_events_write(struct file *file, const char __user *ubuf,
21707f5a08c7SBeau Belgrave size_t count, loff_t *ppos)
21717f5a08c7SBeau Belgrave {
21727f5a08c7SBeau Belgrave struct iovec iov;
21737f5a08c7SBeau Belgrave struct iov_iter i;
21747f5a08c7SBeau Belgrave
21757f5a08c7SBeau Belgrave if (unlikely(*ppos != 0))
21767f5a08c7SBeau Belgrave return -EFAULT;
21777f5a08c7SBeau Belgrave
2178de4eda9dSAl Viro if (unlikely(import_single_range(ITER_SOURCE, (char __user *)ubuf,
217995f18760SBeau Belgrave count, &iov, &i)))
21807f5a08c7SBeau Belgrave return -EFAULT;
21817f5a08c7SBeau Belgrave
21827f5a08c7SBeau Belgrave return user_events_write_core(file, &i);
21837f5a08c7SBeau Belgrave }
21847f5a08c7SBeau Belgrave
user_events_write_iter(struct kiocb * kp,struct iov_iter * i)21857f5a08c7SBeau Belgrave static ssize_t user_events_write_iter(struct kiocb *kp, struct iov_iter *i)
21867f5a08c7SBeau Belgrave {
21877f5a08c7SBeau Belgrave return user_events_write_core(kp->ki_filp, i);
21887f5a08c7SBeau Belgrave }
21897f5a08c7SBeau Belgrave
user_events_ref_add(struct user_event_file_info * info,struct user_event * user)2190e5d27181SBeau Belgrave static int user_events_ref_add(struct user_event_file_info *info,
2191e5d27181SBeau Belgrave struct user_event *user)
21927f5a08c7SBeau Belgrave {
2193e5d27181SBeau Belgrave struct user_event_group *group = info->group;
21947f5a08c7SBeau Belgrave struct user_event_refs *refs, *new_refs;
21957f5a08c7SBeau Belgrave int i, size, count = 0;
21967f5a08c7SBeau Belgrave
2197e5d27181SBeau Belgrave refs = rcu_dereference_protected(info->refs,
2198e5d27181SBeau Belgrave lockdep_is_held(&group->reg_mutex));
21997f5a08c7SBeau Belgrave
22007f5a08c7SBeau Belgrave if (refs) {
22017f5a08c7SBeau Belgrave count = refs->count;
22027f5a08c7SBeau Belgrave
22037f5a08c7SBeau Belgrave for (i = 0; i < count; ++i)
22047f5a08c7SBeau Belgrave if (refs->events[i] == user)
22057f5a08c7SBeau Belgrave return i;
22067f5a08c7SBeau Belgrave }
22077f5a08c7SBeau Belgrave
22087f5a08c7SBeau Belgrave size = struct_size(refs, events, count + 1);
22097f5a08c7SBeau Belgrave
2210f9cce238SBeau Belgrave new_refs = kzalloc(size, GFP_KERNEL_ACCOUNT);
22117f5a08c7SBeau Belgrave
22127f5a08c7SBeau Belgrave if (!new_refs)
22137f5a08c7SBeau Belgrave return -ENOMEM;
22147f5a08c7SBeau Belgrave
22157f5a08c7SBeau Belgrave new_refs->count = count + 1;
22167f5a08c7SBeau Belgrave
22177f5a08c7SBeau Belgrave for (i = 0; i < count; ++i)
22187f5a08c7SBeau Belgrave new_refs->events[i] = refs->events[i];
22197f5a08c7SBeau Belgrave
2220f0dbf6fdSBeau Belgrave new_refs->events[i] = user_event_get(user);
22217f5a08c7SBeau Belgrave
2222e5d27181SBeau Belgrave rcu_assign_pointer(info->refs, new_refs);
22237f5a08c7SBeau Belgrave
22247f5a08c7SBeau Belgrave if (refs)
22257f5a08c7SBeau Belgrave kfree_rcu(refs, rcu);
22267f5a08c7SBeau Belgrave
22277f5a08c7SBeau Belgrave return i;
22287f5a08c7SBeau Belgrave }
22297f5a08c7SBeau Belgrave
user_reg_get(struct user_reg __user * ureg,struct user_reg * kreg)22307f5a08c7SBeau Belgrave static long user_reg_get(struct user_reg __user *ureg, struct user_reg *kreg)
22317f5a08c7SBeau Belgrave {
22327f5a08c7SBeau Belgrave u32 size;
22337f5a08c7SBeau Belgrave long ret;
22347f5a08c7SBeau Belgrave
22357f5a08c7SBeau Belgrave ret = get_user(size, &ureg->size);
22367f5a08c7SBeau Belgrave
22377f5a08c7SBeau Belgrave if (ret)
22387f5a08c7SBeau Belgrave return ret;
22397f5a08c7SBeau Belgrave
22407f5a08c7SBeau Belgrave if (size > PAGE_SIZE)
22417f5a08c7SBeau Belgrave return -E2BIG;
22427f5a08c7SBeau Belgrave
224339d6d08bSBeau Belgrave if (size < offsetofend(struct user_reg, write_index))
224439d6d08bSBeau Belgrave return -EINVAL;
224539d6d08bSBeau Belgrave
224639d6d08bSBeau Belgrave ret = copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
224739d6d08bSBeau Belgrave
224839d6d08bSBeau Belgrave if (ret)
224939d6d08bSBeau Belgrave return ret;
225039d6d08bSBeau Belgrave
2251a65442edSBeau Belgrave /* Ensure only valid flags */
2252a65442edSBeau Belgrave if (kreg->flags & ~(USER_EVENT_REG_MAX-1))
225372357590SBeau Belgrave return -EINVAL;
225472357590SBeau Belgrave
225572357590SBeau Belgrave /* Ensure supported size */
225672357590SBeau Belgrave switch (kreg->enable_size) {
225772357590SBeau Belgrave case 4:
225872357590SBeau Belgrave /* 32-bit */
225972357590SBeau Belgrave break;
226072357590SBeau Belgrave #if BITS_PER_LONG >= 64
226172357590SBeau Belgrave case 8:
226272357590SBeau Belgrave /* 64-bit */
226372357590SBeau Belgrave break;
226472357590SBeau Belgrave #endif
226572357590SBeau Belgrave default:
226672357590SBeau Belgrave return -EINVAL;
226772357590SBeau Belgrave }
226872357590SBeau Belgrave
226972357590SBeau Belgrave /* Ensure natural alignment */
227072357590SBeau Belgrave if (kreg->enable_addr % kreg->enable_size)
227172357590SBeau Belgrave return -EINVAL;
227272357590SBeau Belgrave
227372357590SBeau Belgrave /* Ensure bit range for size */
227472357590SBeau Belgrave if (kreg->enable_bit > (kreg->enable_size * BITS_PER_BYTE) - 1)
227572357590SBeau Belgrave return -EINVAL;
227672357590SBeau Belgrave
227772357590SBeau Belgrave /* Ensure accessible */
227872357590SBeau Belgrave if (!access_ok((const void __user *)(uintptr_t)kreg->enable_addr,
227972357590SBeau Belgrave kreg->enable_size))
228072357590SBeau Belgrave return -EFAULT;
228172357590SBeau Belgrave
228239d6d08bSBeau Belgrave kreg->size = size;
228339d6d08bSBeau Belgrave
228439d6d08bSBeau Belgrave return 0;
22857f5a08c7SBeau Belgrave }
22867f5a08c7SBeau Belgrave
22877f5a08c7SBeau Belgrave /*
22887f5a08c7SBeau Belgrave * Registers a user_event on behalf of a user process.
22897f5a08c7SBeau Belgrave */
user_events_ioctl_reg(struct user_event_file_info * info,unsigned long uarg)2290e5d27181SBeau Belgrave static long user_events_ioctl_reg(struct user_event_file_info *info,
2291e5d27181SBeau Belgrave unsigned long uarg)
22927f5a08c7SBeau Belgrave {
22937f5a08c7SBeau Belgrave struct user_reg __user *ureg = (struct user_reg __user *)uarg;
22947f5a08c7SBeau Belgrave struct user_reg reg;
22957f5a08c7SBeau Belgrave struct user_event *user;
229672357590SBeau Belgrave struct user_event_enabler *enabler;
22977f5a08c7SBeau Belgrave char *name;
22987f5a08c7SBeau Belgrave long ret;
229972357590SBeau Belgrave int write_result;
23007f5a08c7SBeau Belgrave
23017f5a08c7SBeau Belgrave ret = user_reg_get(ureg, ®);
23027f5a08c7SBeau Belgrave
23037f5a08c7SBeau Belgrave if (ret)
23047f5a08c7SBeau Belgrave return ret;
23057f5a08c7SBeau Belgrave
230697bbce89SBeau Belgrave /*
230797bbce89SBeau Belgrave * Prevent users from using the same address and bit multiple times
230897bbce89SBeau Belgrave * within the same mm address space. This can cause unexpected behavior
230997bbce89SBeau Belgrave * for user processes that is far easier to debug if this is explictly
231097bbce89SBeau Belgrave * an error upon registering.
231197bbce89SBeau Belgrave */
231297bbce89SBeau Belgrave if (current_user_event_enabler_exists((unsigned long)reg.enable_addr,
231397bbce89SBeau Belgrave reg.enable_bit))
231497bbce89SBeau Belgrave return -EADDRINUSE;
231597bbce89SBeau Belgrave
23167f5a08c7SBeau Belgrave name = strndup_user((const char __user *)(uintptr_t)reg.name_args,
23177f5a08c7SBeau Belgrave MAX_EVENT_DESC);
23187f5a08c7SBeau Belgrave
23197f5a08c7SBeau Belgrave if (IS_ERR(name)) {
23207f5a08c7SBeau Belgrave ret = PTR_ERR(name);
23217f5a08c7SBeau Belgrave return ret;
23227f5a08c7SBeau Belgrave }
23237f5a08c7SBeau Belgrave
2324b08d7258SBeau Belgrave ret = user_event_parse_cmd(info->group, name, &user, reg.flags);
23257f5a08c7SBeau Belgrave
23267f5a08c7SBeau Belgrave if (ret) {
23277f5a08c7SBeau Belgrave kfree(name);
23287f5a08c7SBeau Belgrave return ret;
23297f5a08c7SBeau Belgrave }
23307f5a08c7SBeau Belgrave
2331e5d27181SBeau Belgrave ret = user_events_ref_add(info, user);
23327f5a08c7SBeau Belgrave
23337e348b32SBeau Belgrave /* No longer need parse ref, ref_add either worked or not */
2334f0dbf6fdSBeau Belgrave user_event_put(user, false);
23357e348b32SBeau Belgrave
23367f5a08c7SBeau Belgrave /* Positive number is index and valid */
23377f5a08c7SBeau Belgrave if (ret < 0)
23387f5a08c7SBeau Belgrave return ret;
23397f5a08c7SBeau Belgrave
234072357590SBeau Belgrave /*
234172357590SBeau Belgrave * user_events_ref_add succeeded:
234272357590SBeau Belgrave * At this point we have a user_event, it's lifetime is bound by the
234372357590SBeau Belgrave * reference count, not this file. If anything fails, the user_event
234472357590SBeau Belgrave * still has a reference until the file is released. During release
234572357590SBeau Belgrave * any remaining references (from user_events_ref_add) are decremented.
234672357590SBeau Belgrave *
234772357590SBeau Belgrave * Attempt to create an enabler, which too has a lifetime tied in the
234872357590SBeau Belgrave * same way for the event. Once the task that caused the enabler to be
234972357590SBeau Belgrave * created exits or issues exec() then the enablers it has created
235072357590SBeau Belgrave * will be destroyed and the ref to the event will be decremented.
235172357590SBeau Belgrave */
235272357590SBeau Belgrave enabler = user_event_enabler_create(®, user, &write_result);
235372357590SBeau Belgrave
235472357590SBeau Belgrave if (!enabler)
235572357590SBeau Belgrave return -ENOMEM;
235672357590SBeau Belgrave
235772357590SBeau Belgrave /* Write failed/faulted, give error back to caller */
235872357590SBeau Belgrave if (write_result)
235972357590SBeau Belgrave return write_result;
236072357590SBeau Belgrave
23617f5a08c7SBeau Belgrave put_user((u32)ret, &ureg->write_index);
23627f5a08c7SBeau Belgrave
23637f5a08c7SBeau Belgrave return 0;
23647f5a08c7SBeau Belgrave }
23657f5a08c7SBeau Belgrave
23667f5a08c7SBeau Belgrave /*
23677f5a08c7SBeau Belgrave * Deletes a user_event on behalf of a user process.
23687f5a08c7SBeau Belgrave */
user_events_ioctl_del(struct user_event_file_info * info,unsigned long uarg)2369e5d27181SBeau Belgrave static long user_events_ioctl_del(struct user_event_file_info *info,
2370e5d27181SBeau Belgrave unsigned long uarg)
23717f5a08c7SBeau Belgrave {
23727f5a08c7SBeau Belgrave void __user *ubuf = (void __user *)uarg;
23737f5a08c7SBeau Belgrave char *name;
23747f5a08c7SBeau Belgrave long ret;
23757f5a08c7SBeau Belgrave
23767f5a08c7SBeau Belgrave name = strndup_user(ubuf, MAX_EVENT_DESC);
23777f5a08c7SBeau Belgrave
23787f5a08c7SBeau Belgrave if (IS_ERR(name))
23797f5a08c7SBeau Belgrave return PTR_ERR(name);
23807f5a08c7SBeau Belgrave
23817e348b32SBeau Belgrave /* event_mutex prevents dyn_event from racing */
23827e348b32SBeau Belgrave mutex_lock(&event_mutex);
2383e5d27181SBeau Belgrave ret = delete_user_event(info->group, name);
23847e348b32SBeau Belgrave mutex_unlock(&event_mutex);
23857f5a08c7SBeau Belgrave
23867f5a08c7SBeau Belgrave kfree(name);
23877f5a08c7SBeau Belgrave
23887f5a08c7SBeau Belgrave return ret;
23897f5a08c7SBeau Belgrave }
23907f5a08c7SBeau Belgrave
user_unreg_get(struct user_unreg __user * ureg,struct user_unreg * kreg)2391dcb8177cSBeau Belgrave static long user_unreg_get(struct user_unreg __user *ureg,
2392dcb8177cSBeau Belgrave struct user_unreg *kreg)
2393dcb8177cSBeau Belgrave {
2394dcb8177cSBeau Belgrave u32 size;
2395dcb8177cSBeau Belgrave long ret;
2396dcb8177cSBeau Belgrave
2397dcb8177cSBeau Belgrave ret = get_user(size, &ureg->size);
2398dcb8177cSBeau Belgrave
2399dcb8177cSBeau Belgrave if (ret)
2400dcb8177cSBeau Belgrave return ret;
2401dcb8177cSBeau Belgrave
2402dcb8177cSBeau Belgrave if (size > PAGE_SIZE)
2403dcb8177cSBeau Belgrave return -E2BIG;
2404dcb8177cSBeau Belgrave
2405dcb8177cSBeau Belgrave if (size < offsetofend(struct user_unreg, disable_addr))
2406dcb8177cSBeau Belgrave return -EINVAL;
2407dcb8177cSBeau Belgrave
2408dcb8177cSBeau Belgrave ret = copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
2409dcb8177cSBeau Belgrave
2410dcb8177cSBeau Belgrave /* Ensure no reserved values, since we don't support any yet */
2411dcb8177cSBeau Belgrave if (kreg->__reserved || kreg->__reserved2)
2412dcb8177cSBeau Belgrave return -EINVAL;
2413dcb8177cSBeau Belgrave
2414dcb8177cSBeau Belgrave return ret;
2415dcb8177cSBeau Belgrave }
2416dcb8177cSBeau Belgrave
user_event_mm_clear_bit(struct user_event_mm * user_mm,unsigned long uaddr,unsigned char bit,unsigned long flags)241717b439dbSBeau Belgrave static int user_event_mm_clear_bit(struct user_event_mm *user_mm,
2418*2de9ee94SBeau Belgrave unsigned long uaddr, unsigned char bit,
2419*2de9ee94SBeau Belgrave unsigned long flags)
242017b439dbSBeau Belgrave {
242117b439dbSBeau Belgrave struct user_event_enabler enabler;
242217b439dbSBeau Belgrave int result;
242341d8fba1SBeau Belgrave int attempt = 0;
242417b439dbSBeau Belgrave
242517b439dbSBeau Belgrave memset(&enabler, 0, sizeof(enabler));
242617b439dbSBeau Belgrave enabler.addr = uaddr;
2427*2de9ee94SBeau Belgrave enabler.values = bit | flags;
242817b439dbSBeau Belgrave retry:
242917b439dbSBeau Belgrave /* Prevents state changes from racing with new enablers */
243017b439dbSBeau Belgrave mutex_lock(&event_mutex);
243117b439dbSBeau Belgrave
243217b439dbSBeau Belgrave /* Force the bit to be cleared, since no event is attached */
243317b439dbSBeau Belgrave mmap_read_lock(user_mm->mm);
243441d8fba1SBeau Belgrave result = user_event_enabler_write(user_mm, &enabler, false, &attempt);
243517b439dbSBeau Belgrave mmap_read_unlock(user_mm->mm);
243617b439dbSBeau Belgrave
243717b439dbSBeau Belgrave mutex_unlock(&event_mutex);
243817b439dbSBeau Belgrave
243917b439dbSBeau Belgrave if (result) {
244017b439dbSBeau Belgrave /* Attempt to fault-in and retry if it worked */
244141d8fba1SBeau Belgrave if (!user_event_mm_fault_in(user_mm, uaddr, attempt))
244217b439dbSBeau Belgrave goto retry;
244317b439dbSBeau Belgrave }
244417b439dbSBeau Belgrave
244517b439dbSBeau Belgrave return result;
244617b439dbSBeau Belgrave }
244717b439dbSBeau Belgrave
2448dcb8177cSBeau Belgrave /*
2449dcb8177cSBeau Belgrave * Unregisters an enablement address/bit within a task/user mm.
2450dcb8177cSBeau Belgrave */
user_events_ioctl_unreg(unsigned long uarg)2451dcb8177cSBeau Belgrave static long user_events_ioctl_unreg(unsigned long uarg)
2452dcb8177cSBeau Belgrave {
2453dcb8177cSBeau Belgrave struct user_unreg __user *ureg = (struct user_unreg __user *)uarg;
2454dcb8177cSBeau Belgrave struct user_event_mm *mm = current->user_event_mm;
2455dcb8177cSBeau Belgrave struct user_event_enabler *enabler, *next;
2456dcb8177cSBeau Belgrave struct user_unreg reg;
2457*2de9ee94SBeau Belgrave unsigned long flags;
2458dcb8177cSBeau Belgrave long ret;
2459dcb8177cSBeau Belgrave
2460dcb8177cSBeau Belgrave ret = user_unreg_get(ureg, ®);
2461dcb8177cSBeau Belgrave
2462dcb8177cSBeau Belgrave if (ret)
2463dcb8177cSBeau Belgrave return ret;
2464dcb8177cSBeau Belgrave
2465dcb8177cSBeau Belgrave if (!mm)
2466dcb8177cSBeau Belgrave return -ENOENT;
2467dcb8177cSBeau Belgrave
2468*2de9ee94SBeau Belgrave flags = 0;
2469dcb8177cSBeau Belgrave ret = -ENOENT;
2470dcb8177cSBeau Belgrave
2471dcb8177cSBeau Belgrave /*
2472dcb8177cSBeau Belgrave * Flags freeing and faulting are used to indicate if the enabler is in
2473dcb8177cSBeau Belgrave * use at all. When faulting is set a page-fault is occurring asyncly.
2474dcb8177cSBeau Belgrave * During async fault if freeing is set, the enabler will be destroyed.
2475dcb8177cSBeau Belgrave * If no async fault is happening, we can destroy it now since we hold
2476dcb8177cSBeau Belgrave * the event_mutex during these checks.
2477dcb8177cSBeau Belgrave */
2478dcb8177cSBeau Belgrave mutex_lock(&event_mutex);
2479dcb8177cSBeau Belgrave
2480dcbd1ac2SBeau Belgrave list_for_each_entry_safe(enabler, next, &mm->enablers, mm_enablers_link) {
2481dcb8177cSBeau Belgrave if (enabler->addr == reg.disable_addr &&
2482ee7751b5SBeau Belgrave ENABLE_BIT(enabler) == reg.disable_bit) {
2483dcb8177cSBeau Belgrave set_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler));
2484dcb8177cSBeau Belgrave
2485*2de9ee94SBeau Belgrave /* We must keep compat flags for the clear */
2486*2de9ee94SBeau Belgrave flags |= enabler->values & ENABLE_VAL_COMPAT_MASK;
2487*2de9ee94SBeau Belgrave
2488dcb8177cSBeau Belgrave if (!test_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler)))
2489f0dbf6fdSBeau Belgrave user_event_enabler_destroy(enabler, true);
2490dcb8177cSBeau Belgrave
2491dcb8177cSBeau Belgrave /* Removed at least one */
2492dcb8177cSBeau Belgrave ret = 0;
2493dcb8177cSBeau Belgrave }
2494dcbd1ac2SBeau Belgrave }
2495dcb8177cSBeau Belgrave
2496dcb8177cSBeau Belgrave mutex_unlock(&event_mutex);
2497dcb8177cSBeau Belgrave
249817b439dbSBeau Belgrave /* Ensure bit is now cleared for user, regardless of event status */
249917b439dbSBeau Belgrave if (!ret)
250017b439dbSBeau Belgrave ret = user_event_mm_clear_bit(mm, reg.disable_addr,
2501*2de9ee94SBeau Belgrave reg.disable_bit, flags);
250217b439dbSBeau Belgrave
2503dcb8177cSBeau Belgrave return ret;
2504dcb8177cSBeau Belgrave }
2505dcb8177cSBeau Belgrave
25067f5a08c7SBeau Belgrave /*
25077f5a08c7SBeau Belgrave * Handles the ioctl from user mode to register or alter operations.
25087f5a08c7SBeau Belgrave */
user_events_ioctl(struct file * file,unsigned int cmd,unsigned long uarg)25097f5a08c7SBeau Belgrave static long user_events_ioctl(struct file *file, unsigned int cmd,
25107f5a08c7SBeau Belgrave unsigned long uarg)
25117f5a08c7SBeau Belgrave {
2512e5d27181SBeau Belgrave struct user_event_file_info *info = file->private_data;
2513e5d27181SBeau Belgrave struct user_event_group *group = info->group;
25147f5a08c7SBeau Belgrave long ret = -ENOTTY;
25157f5a08c7SBeau Belgrave
25167f5a08c7SBeau Belgrave switch (cmd) {
25177f5a08c7SBeau Belgrave case DIAG_IOCSREG:
2518e5d27181SBeau Belgrave mutex_lock(&group->reg_mutex);
2519e5d27181SBeau Belgrave ret = user_events_ioctl_reg(info, uarg);
2520e5d27181SBeau Belgrave mutex_unlock(&group->reg_mutex);
25217f5a08c7SBeau Belgrave break;
25227f5a08c7SBeau Belgrave
25237f5a08c7SBeau Belgrave case DIAG_IOCSDEL:
2524e5d27181SBeau Belgrave mutex_lock(&group->reg_mutex);
2525e5d27181SBeau Belgrave ret = user_events_ioctl_del(info, uarg);
2526e5d27181SBeau Belgrave mutex_unlock(&group->reg_mutex);
25277f5a08c7SBeau Belgrave break;
2528dcb8177cSBeau Belgrave
2529dcb8177cSBeau Belgrave case DIAG_IOCSUNREG:
2530dcb8177cSBeau Belgrave mutex_lock(&group->reg_mutex);
2531dcb8177cSBeau Belgrave ret = user_events_ioctl_unreg(uarg);
2532dcb8177cSBeau Belgrave mutex_unlock(&group->reg_mutex);
2533dcb8177cSBeau Belgrave break;
25347f5a08c7SBeau Belgrave }
25357f5a08c7SBeau Belgrave
25367f5a08c7SBeau Belgrave return ret;
25377f5a08c7SBeau Belgrave }
25387f5a08c7SBeau Belgrave
25397f5a08c7SBeau Belgrave /*
25407f5a08c7SBeau Belgrave * Handles the final close of the file from user mode.
25417f5a08c7SBeau Belgrave */
user_events_release(struct inode * node,struct file * file)25427f5a08c7SBeau Belgrave static int user_events_release(struct inode *node, struct file *file)
25437f5a08c7SBeau Belgrave {
2544e5d27181SBeau Belgrave struct user_event_file_info *info = file->private_data;
2545e5d27181SBeau Belgrave struct user_event_group *group;
25467f5a08c7SBeau Belgrave struct user_event_refs *refs;
25477f5a08c7SBeau Belgrave int i;
25487f5a08c7SBeau Belgrave
2549e5d27181SBeau Belgrave if (!info)
2550e5d27181SBeau Belgrave return -EINVAL;
2551e5d27181SBeau Belgrave
2552e5d27181SBeau Belgrave group = info->group;
2553e5d27181SBeau Belgrave
25547f5a08c7SBeau Belgrave /*
25557f5a08c7SBeau Belgrave * Ensure refs cannot change under any situation by taking the
25567f5a08c7SBeau Belgrave * register mutex during the final freeing of the references.
25577f5a08c7SBeau Belgrave */
2558e5d27181SBeau Belgrave mutex_lock(&group->reg_mutex);
25597f5a08c7SBeau Belgrave
2560e5d27181SBeau Belgrave refs = info->refs;
25617f5a08c7SBeau Belgrave
25627f5a08c7SBeau Belgrave if (!refs)
25637f5a08c7SBeau Belgrave goto out;
25647f5a08c7SBeau Belgrave
25657f5a08c7SBeau Belgrave /*
25667f5a08c7SBeau Belgrave * The lifetime of refs has reached an end, it's tied to this file.
25677f5a08c7SBeau Belgrave * The underlying user_events are ref counted, and cannot be freed.
25687f5a08c7SBeau Belgrave * After this decrement, the user_events may be freed elsewhere.
25697f5a08c7SBeau Belgrave */
2570f0dbf6fdSBeau Belgrave for (i = 0; i < refs->count; ++i)
2571f0dbf6fdSBeau Belgrave user_event_put(refs->events[i], false);
25727f5a08c7SBeau Belgrave
25737f5a08c7SBeau Belgrave out:
25747f5a08c7SBeau Belgrave file->private_data = NULL;
25757f5a08c7SBeau Belgrave
2576e5d27181SBeau Belgrave mutex_unlock(&group->reg_mutex);
25777f5a08c7SBeau Belgrave
25787f5a08c7SBeau Belgrave kfree(refs);
2579e5d27181SBeau Belgrave kfree(info);
25807f5a08c7SBeau Belgrave
25817f5a08c7SBeau Belgrave return 0;
25827f5a08c7SBeau Belgrave }
25837f5a08c7SBeau Belgrave
25847f5a08c7SBeau Belgrave static const struct file_operations user_data_fops = {
2585e5d27181SBeau Belgrave .open = user_events_open,
25867f5a08c7SBeau Belgrave .write = user_events_write,
25877f5a08c7SBeau Belgrave .write_iter = user_events_write_iter,
25887f5a08c7SBeau Belgrave .unlocked_ioctl = user_events_ioctl,
25897f5a08c7SBeau Belgrave .release = user_events_release,
25907f5a08c7SBeau Belgrave };
25917f5a08c7SBeau Belgrave
user_seq_start(struct seq_file * m,loff_t * pos)25927f5a08c7SBeau Belgrave static void *user_seq_start(struct seq_file *m, loff_t *pos)
25937f5a08c7SBeau Belgrave {
25947f5a08c7SBeau Belgrave if (*pos)
25957f5a08c7SBeau Belgrave return NULL;
25967f5a08c7SBeau Belgrave
25977f5a08c7SBeau Belgrave return (void *)1;
25987f5a08c7SBeau Belgrave }
25997f5a08c7SBeau Belgrave
user_seq_next(struct seq_file * m,void * p,loff_t * pos)26007f5a08c7SBeau Belgrave static void *user_seq_next(struct seq_file *m, void *p, loff_t *pos)
26017f5a08c7SBeau Belgrave {
26027f5a08c7SBeau Belgrave ++*pos;
26037f5a08c7SBeau Belgrave return NULL;
26047f5a08c7SBeau Belgrave }
26057f5a08c7SBeau Belgrave
user_seq_stop(struct seq_file * m,void * p)26067f5a08c7SBeau Belgrave static void user_seq_stop(struct seq_file *m, void *p)
26077f5a08c7SBeau Belgrave {
26087f5a08c7SBeau Belgrave }
26097f5a08c7SBeau Belgrave
user_seq_show(struct seq_file * m,void * p)26107f5a08c7SBeau Belgrave static int user_seq_show(struct seq_file *m, void *p)
26117f5a08c7SBeau Belgrave {
2612e5d27181SBeau Belgrave struct user_event_group *group = m->private;
26137f5a08c7SBeau Belgrave struct user_event *user;
26147f5a08c7SBeau Belgrave char status;
261572357590SBeau Belgrave int i, active = 0, busy = 0;
26167f5a08c7SBeau Belgrave
2617e5d27181SBeau Belgrave if (!group)
2618e5d27181SBeau Belgrave return -EINVAL;
26197f5a08c7SBeau Belgrave
2620e5d27181SBeau Belgrave mutex_lock(&group->reg_mutex);
2621e5d27181SBeau Belgrave
2622e5d27181SBeau Belgrave hash_for_each(group->register_table, i, user, node) {
262339d6d08bSBeau Belgrave status = user->status;
26247f5a08c7SBeau Belgrave
262572357590SBeau Belgrave seq_printf(m, "%s", EVENT_NAME(user));
26267f5a08c7SBeau Belgrave
262772357590SBeau Belgrave if (status != 0)
26287f5a08c7SBeau Belgrave seq_puts(m, " #");
26297f5a08c7SBeau Belgrave
26307f5a08c7SBeau Belgrave if (status != 0) {
26317f5a08c7SBeau Belgrave seq_puts(m, " Used by");
26327f5a08c7SBeau Belgrave if (status & EVENT_STATUS_FTRACE)
26337f5a08c7SBeau Belgrave seq_puts(m, " ftrace");
26347f5a08c7SBeau Belgrave if (status & EVENT_STATUS_PERF)
26357f5a08c7SBeau Belgrave seq_puts(m, " perf");
26367f5a08c7SBeau Belgrave if (status & EVENT_STATUS_OTHER)
26377f5a08c7SBeau Belgrave seq_puts(m, " other");
26387f5a08c7SBeau Belgrave busy++;
26397f5a08c7SBeau Belgrave }
26407f5a08c7SBeau Belgrave
26417f5a08c7SBeau Belgrave seq_puts(m, "\n");
26427f5a08c7SBeau Belgrave active++;
26437f5a08c7SBeau Belgrave }
26447f5a08c7SBeau Belgrave
2645e5d27181SBeau Belgrave mutex_unlock(&group->reg_mutex);
26467f5a08c7SBeau Belgrave
26477f5a08c7SBeau Belgrave seq_puts(m, "\n");
26487f5a08c7SBeau Belgrave seq_printf(m, "Active: %d\n", active);
26497f5a08c7SBeau Belgrave seq_printf(m, "Busy: %d\n", busy);
26507f5a08c7SBeau Belgrave
26517f5a08c7SBeau Belgrave return 0;
26527f5a08c7SBeau Belgrave }
26537f5a08c7SBeau Belgrave
26547f5a08c7SBeau Belgrave static const struct seq_operations user_seq_ops = {
26557f5a08c7SBeau Belgrave .start = user_seq_start,
26567f5a08c7SBeau Belgrave .next = user_seq_next,
26577f5a08c7SBeau Belgrave .stop = user_seq_stop,
26587f5a08c7SBeau Belgrave .show = user_seq_show,
26597f5a08c7SBeau Belgrave };
26607f5a08c7SBeau Belgrave
user_status_open(struct inode * node,struct file * file)26617f5a08c7SBeau Belgrave static int user_status_open(struct inode *node, struct file *file)
26627f5a08c7SBeau Belgrave {
2663e5d27181SBeau Belgrave struct user_event_group *group;
2664e5d27181SBeau Belgrave int ret;
2665e5d27181SBeau Belgrave
2666e5d27181SBeau Belgrave group = current_user_event_group();
2667e5d27181SBeau Belgrave
2668e5d27181SBeau Belgrave if (!group)
2669e5d27181SBeau Belgrave return -ENOENT;
2670e5d27181SBeau Belgrave
2671e5d27181SBeau Belgrave ret = seq_open(file, &user_seq_ops);
2672e5d27181SBeau Belgrave
2673e5d27181SBeau Belgrave if (!ret) {
2674e5d27181SBeau Belgrave /* Chain group to seq_file */
2675e5d27181SBeau Belgrave struct seq_file *m = file->private_data;
2676e5d27181SBeau Belgrave
2677e5d27181SBeau Belgrave m->private = group;
2678e5d27181SBeau Belgrave }
2679e5d27181SBeau Belgrave
2680e5d27181SBeau Belgrave return ret;
26817f5a08c7SBeau Belgrave }
26827f5a08c7SBeau Belgrave
26837f5a08c7SBeau Belgrave static const struct file_operations user_status_fops = {
26847f5a08c7SBeau Belgrave .open = user_status_open,
26857f5a08c7SBeau Belgrave .read = seq_read,
26867f5a08c7SBeau Belgrave .llseek = seq_lseek,
26877f5a08c7SBeau Belgrave .release = seq_release,
26887f5a08c7SBeau Belgrave };
26897f5a08c7SBeau Belgrave
26907f5a08c7SBeau Belgrave /*
26917f5a08c7SBeau Belgrave * Creates a set of tracefs files to allow user mode interactions.
26927f5a08c7SBeau Belgrave */
create_user_tracefs(void)26937f5a08c7SBeau Belgrave static int create_user_tracefs(void)
26947f5a08c7SBeau Belgrave {
26957f5a08c7SBeau Belgrave struct dentry *edata, *emmap;
26967f5a08c7SBeau Belgrave
26977f5a08c7SBeau Belgrave edata = tracefs_create_file("user_events_data", TRACE_MODE_WRITE,
26987f5a08c7SBeau Belgrave NULL, NULL, &user_data_fops);
26997f5a08c7SBeau Belgrave
27007f5a08c7SBeau Belgrave if (!edata) {
27017f5a08c7SBeau Belgrave pr_warn("Could not create tracefs 'user_events_data' entry\n");
27027f5a08c7SBeau Belgrave goto err;
27037f5a08c7SBeau Belgrave }
27047f5a08c7SBeau Belgrave
270572357590SBeau Belgrave emmap = tracefs_create_file("user_events_status", TRACE_MODE_READ,
27067f5a08c7SBeau Belgrave NULL, NULL, &user_status_fops);
27077f5a08c7SBeau Belgrave
27087f5a08c7SBeau Belgrave if (!emmap) {
27097f5a08c7SBeau Belgrave tracefs_remove(edata);
27107f5a08c7SBeau Belgrave pr_warn("Could not create tracefs 'user_events_mmap' entry\n");
27117f5a08c7SBeau Belgrave goto err;
27127f5a08c7SBeau Belgrave }
27137f5a08c7SBeau Belgrave
27147f5a08c7SBeau Belgrave return 0;
27157f5a08c7SBeau Belgrave err:
27167f5a08c7SBeau Belgrave return -ENODEV;
27177f5a08c7SBeau Belgrave }
27187f5a08c7SBeau Belgrave
set_max_user_events_sysctl(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)2719ce58e96eSBeau Belgrave static int set_max_user_events_sysctl(struct ctl_table *table, int write,
2720ce58e96eSBeau Belgrave void *buffer, size_t *lenp, loff_t *ppos)
2721ce58e96eSBeau Belgrave {
2722ce58e96eSBeau Belgrave int ret;
2723ce58e96eSBeau Belgrave
2724ce58e96eSBeau Belgrave mutex_lock(&event_mutex);
2725ce58e96eSBeau Belgrave
2726ce58e96eSBeau Belgrave ret = proc_douintvec(table, write, buffer, lenp, ppos);
2727ce58e96eSBeau Belgrave
2728ce58e96eSBeau Belgrave mutex_unlock(&event_mutex);
2729ce58e96eSBeau Belgrave
2730ce58e96eSBeau Belgrave return ret;
2731ce58e96eSBeau Belgrave }
2732ce58e96eSBeau Belgrave
2733ce58e96eSBeau Belgrave static struct ctl_table user_event_sysctls[] = {
2734ce58e96eSBeau Belgrave {
2735ce58e96eSBeau Belgrave .procname = "user_events_max",
2736ce58e96eSBeau Belgrave .data = &max_user_events,
2737ce58e96eSBeau Belgrave .maxlen = sizeof(unsigned int),
2738ce58e96eSBeau Belgrave .mode = 0644,
2739ce58e96eSBeau Belgrave .proc_handler = set_max_user_events_sysctl,
2740ce58e96eSBeau Belgrave },
2741ce58e96eSBeau Belgrave {}
2742ce58e96eSBeau Belgrave };
2743ce58e96eSBeau Belgrave
trace_events_user_init(void)27447f5a08c7SBeau Belgrave static int __init trace_events_user_init(void)
27457f5a08c7SBeau Belgrave {
27467f5a08c7SBeau Belgrave int ret;
27477f5a08c7SBeau Belgrave
274881f8fb65SBeau Belgrave fault_cache = KMEM_CACHE(user_event_enabler_fault, 0);
274981f8fb65SBeau Belgrave
275081f8fb65SBeau Belgrave if (!fault_cache)
275181f8fb65SBeau Belgrave return -ENOMEM;
275281f8fb65SBeau Belgrave
2753ed0e0ae0SBeau Belgrave init_group = user_event_group_create();
27547f5a08c7SBeau Belgrave
275581f8fb65SBeau Belgrave if (!init_group) {
275681f8fb65SBeau Belgrave kmem_cache_destroy(fault_cache);
27577f5a08c7SBeau Belgrave return -ENOMEM;
275881f8fb65SBeau Belgrave }
27597f5a08c7SBeau Belgrave
27607f5a08c7SBeau Belgrave ret = create_user_tracefs();
27617f5a08c7SBeau Belgrave
27627f5a08c7SBeau Belgrave if (ret) {
27637f5a08c7SBeau Belgrave pr_warn("user_events could not register with tracefs\n");
2764e5d27181SBeau Belgrave user_event_group_destroy(init_group);
276581f8fb65SBeau Belgrave kmem_cache_destroy(fault_cache);
2766e5d27181SBeau Belgrave init_group = NULL;
27677f5a08c7SBeau Belgrave return ret;
27687f5a08c7SBeau Belgrave }
27697f5a08c7SBeau Belgrave
27707f5a08c7SBeau Belgrave if (dyn_event_register(&user_event_dops))
27717f5a08c7SBeau Belgrave pr_warn("user_events could not register with dyn_events\n");
27727f5a08c7SBeau Belgrave
2773ce58e96eSBeau Belgrave register_sysctl_init("kernel", user_event_sysctls);
2774ce58e96eSBeau Belgrave
27757f5a08c7SBeau Belgrave return 0;
27767f5a08c7SBeau Belgrave }
27777f5a08c7SBeau Belgrave
27787f5a08c7SBeau Belgrave fs_initcall(trace_events_user_init);
2779