12b5067a8SAxel Rasmussen // SPDX-License-Identifier: GPL-2.0
22b5067a8SAxel Rasmussen #define CREATE_TRACE_POINTS
32b5067a8SAxel Rasmussen #include <trace/events/mmap_lock.h>
42b5067a8SAxel Rasmussen
52b5067a8SAxel Rasmussen #include <linux/mm.h>
62b5067a8SAxel Rasmussen #include <linux/cgroup.h>
72b5067a8SAxel Rasmussen #include <linux/memcontrol.h>
82b5067a8SAxel Rasmussen #include <linux/mmap_lock.h>
92b5067a8SAxel Rasmussen #include <linux/mutex.h>
102b5067a8SAxel Rasmussen #include <linux/percpu.h>
112b5067a8SAxel Rasmussen #include <linux/rcupdate.h>
122b5067a8SAxel Rasmussen #include <linux/smp.h>
132b5067a8SAxel Rasmussen #include <linux/trace_events.h>
14832b5072SNicolas Saenz Julienne #include <linux/local_lock.h>
152b5067a8SAxel Rasmussen
162b5067a8SAxel Rasmussen EXPORT_TRACEPOINT_SYMBOL(mmap_lock_start_locking);
172b5067a8SAxel Rasmussen EXPORT_TRACEPOINT_SYMBOL(mmap_lock_acquire_returned);
182b5067a8SAxel Rasmussen EXPORT_TRACEPOINT_SYMBOL(mmap_lock_released);
192b5067a8SAxel Rasmussen
202b5067a8SAxel Rasmussen #ifdef CONFIG_MEMCG
212b5067a8SAxel Rasmussen
22*0038abf9STetsuo Handa static atomic_t reg_refcount;
232b5067a8SAxel Rasmussen
242b5067a8SAxel Rasmussen /*
252b5067a8SAxel Rasmussen * Size of the buffer for memcg path names. Ignoring stack trace support,
262b5067a8SAxel Rasmussen * trace_events_hist.c uses MAX_FILTER_STR_VAL for this, so we also use it.
272b5067a8SAxel Rasmussen */
282b5067a8SAxel Rasmussen #define MEMCG_PATH_BUF_SIZE MAX_FILTER_STR_VAL
292b5067a8SAxel Rasmussen
trace_mmap_lock_reg(void)302b5067a8SAxel Rasmussen int trace_mmap_lock_reg(void)
312b5067a8SAxel Rasmussen {
32*0038abf9STetsuo Handa atomic_inc(®_refcount);
332b5067a8SAxel Rasmussen return 0;
342b5067a8SAxel Rasmussen }
352b5067a8SAxel Rasmussen
trace_mmap_lock_unreg(void)362b5067a8SAxel Rasmussen void trace_mmap_lock_unreg(void)
372b5067a8SAxel Rasmussen {
38*0038abf9STetsuo Handa atomic_dec(®_refcount);
392b5067a8SAxel Rasmussen }
402b5067a8SAxel Rasmussen
41d01079f3SMel Gorman #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) \
42d01079f3SMel Gorman do { \
43*0038abf9STetsuo Handa char buf[MEMCG_PATH_BUF_SIZE]; \
44*0038abf9STetsuo Handa get_mm_memcg_path(mm, buf, sizeof(buf)); \
45*0038abf9STetsuo Handa trace_mmap_lock_##type(mm, buf, ##__VA_ARGS__); \
46d01079f3SMel Gorman } while (0)
47d01079f3SMel Gorman
48d01079f3SMel Gorman #else /* !CONFIG_MEMCG */
49d01079f3SMel Gorman
trace_mmap_lock_reg(void)50d01079f3SMel Gorman int trace_mmap_lock_reg(void)
51d01079f3SMel Gorman {
52d01079f3SMel Gorman return 0;
53d01079f3SMel Gorman }
54d01079f3SMel Gorman
trace_mmap_lock_unreg(void)55d01079f3SMel Gorman void trace_mmap_lock_unreg(void)
56d01079f3SMel Gorman {
57d01079f3SMel Gorman }
58d01079f3SMel Gorman
59d01079f3SMel Gorman #define TRACE_MMAP_LOCK_EVENT(type, mm, ...) \
60d01079f3SMel Gorman trace_mmap_lock_##type(mm, "", ##__VA_ARGS__)
61d01079f3SMel Gorman
62d01079f3SMel Gorman #endif /* CONFIG_MEMCG */
63d01079f3SMel Gorman
64d01079f3SMel Gorman #ifdef CONFIG_TRACING
65d01079f3SMel Gorman #ifdef CONFIG_MEMCG
662b5067a8SAxel Rasmussen /*
67*0038abf9STetsuo Handa * Write the given mm_struct's memcg path to a buffer. If the path cannot be
68*0038abf9STetsuo Handa * determined or the trace event is being unregistered, empty string is written.
692b5067a8SAxel Rasmussen */
get_mm_memcg_path(struct mm_struct * mm,char * buf,size_t buflen)70*0038abf9STetsuo Handa static void get_mm_memcg_path(struct mm_struct *mm, char *buf, size_t buflen)
712b5067a8SAxel Rasmussen {
72*0038abf9STetsuo Handa struct mem_cgroup *memcg;
732b5067a8SAxel Rasmussen
74*0038abf9STetsuo Handa buf[0] = '\0';
75*0038abf9STetsuo Handa /* No need to get path if no trace event is registered. */
76*0038abf9STetsuo Handa if (!atomic_read(®_refcount))
77*0038abf9STetsuo Handa return;
78*0038abf9STetsuo Handa memcg = get_mem_cgroup_from_mm(mm);
792b5067a8SAxel Rasmussen if (memcg == NULL)
80*0038abf9STetsuo Handa return;
81*0038abf9STetsuo Handa if (memcg->css.cgroup)
82*0038abf9STetsuo Handa cgroup_path(memcg->css.cgroup, buf, buflen);
832b5067a8SAxel Rasmussen css_put(&memcg->css);
842b5067a8SAxel Rasmussen }
852b5067a8SAxel Rasmussen
862b5067a8SAxel Rasmussen #endif /* CONFIG_MEMCG */
872b5067a8SAxel Rasmussen
882b5067a8SAxel Rasmussen /*
892b5067a8SAxel Rasmussen * Trace calls must be in a separate file, as otherwise there's a circular
902b5067a8SAxel Rasmussen * dependency between linux/mmap_lock.h and trace/events/mmap_lock.h.
912b5067a8SAxel Rasmussen */
922b5067a8SAxel Rasmussen
__mmap_lock_do_trace_start_locking(struct mm_struct * mm,bool write)932b5067a8SAxel Rasmussen void __mmap_lock_do_trace_start_locking(struct mm_struct *mm, bool write)
942b5067a8SAxel Rasmussen {
952b5067a8SAxel Rasmussen TRACE_MMAP_LOCK_EVENT(start_locking, mm, write);
962b5067a8SAxel Rasmussen }
972b5067a8SAxel Rasmussen EXPORT_SYMBOL(__mmap_lock_do_trace_start_locking);
982b5067a8SAxel Rasmussen
__mmap_lock_do_trace_acquire_returned(struct mm_struct * mm,bool write,bool success)992b5067a8SAxel Rasmussen void __mmap_lock_do_trace_acquire_returned(struct mm_struct *mm, bool write,
1002b5067a8SAxel Rasmussen bool success)
1012b5067a8SAxel Rasmussen {
1022b5067a8SAxel Rasmussen TRACE_MMAP_LOCK_EVENT(acquire_returned, mm, write, success);
1032b5067a8SAxel Rasmussen }
1042b5067a8SAxel Rasmussen EXPORT_SYMBOL(__mmap_lock_do_trace_acquire_returned);
1052b5067a8SAxel Rasmussen
__mmap_lock_do_trace_released(struct mm_struct * mm,bool write)1062b5067a8SAxel Rasmussen void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write)
1072b5067a8SAxel Rasmussen {
1082b5067a8SAxel Rasmussen TRACE_MMAP_LOCK_EVENT(released, mm, write);
1092b5067a8SAxel Rasmussen }
1102b5067a8SAxel Rasmussen EXPORT_SYMBOL(__mmap_lock_do_trace_released);
111d01079f3SMel Gorman #endif /* CONFIG_TRACING */
112