1bce29ac9SDaniel Bristot de Oliveira // SPDX-License-Identifier: GPL-2.0
2bce29ac9SDaniel Bristot de Oliveira /*
3bce29ac9SDaniel Bristot de Oliveira * OS Noise Tracer: computes the OS Noise suffered by a running thread.
4a955d7eaSDaniel Bristot de Oliveira * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread.
5bce29ac9SDaniel Bristot de Oliveira *
6bce29ac9SDaniel Bristot de Oliveira * Based on "hwlat_detector" tracer by:
7bce29ac9SDaniel Bristot de Oliveira * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
8bce29ac9SDaniel Bristot de Oliveira * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
9bce29ac9SDaniel Bristot de Oliveira * With feedback from Clark Williams <williams@redhat.com>
10bce29ac9SDaniel Bristot de Oliveira *
11bce29ac9SDaniel Bristot de Oliveira * And also based on the rtsl tracer presented on:
12bce29ac9SDaniel Bristot de Oliveira * DE OLIVEIRA, Daniel Bristot, et al. Demystifying the real-time linux
13bce29ac9SDaniel Bristot de Oliveira * scheduling latency. In: 32nd Euromicro Conference on Real-Time Systems
14bce29ac9SDaniel Bristot de Oliveira * (ECRTS 2020). Schloss Dagstuhl-Leibniz-Zentrum fur Informatik, 2020.
15bce29ac9SDaniel Bristot de Oliveira *
16bce29ac9SDaniel Bristot de Oliveira * Copyright (C) 2021 Daniel Bristot de Oliveira, Red Hat, Inc. <bristot@redhat.com>
17bce29ac9SDaniel Bristot de Oliveira */
18bce29ac9SDaniel Bristot de Oliveira
19bce29ac9SDaniel Bristot de Oliveira #include <linux/kthread.h>
20bce29ac9SDaniel Bristot de Oliveira #include <linux/tracefs.h>
21bce29ac9SDaniel Bristot de Oliveira #include <linux/uaccess.h>
22bce29ac9SDaniel Bristot de Oliveira #include <linux/cpumask.h>
23bce29ac9SDaniel Bristot de Oliveira #include <linux/delay.h>
24bce29ac9SDaniel Bristot de Oliveira #include <linux/sched/clock.h>
25a955d7eaSDaniel Bristot de Oliveira #include <uapi/linux/sched/types.h>
26bce29ac9SDaniel Bristot de Oliveira #include <linux/sched.h>
27bce29ac9SDaniel Bristot de Oliveira #include "trace.h"
28bce29ac9SDaniel Bristot de Oliveira
29bce29ac9SDaniel Bristot de Oliveira #ifdef CONFIG_X86_LOCAL_APIC
30bce29ac9SDaniel Bristot de Oliveira #include <asm/trace/irq_vectors.h>
31bce29ac9SDaniel Bristot de Oliveira #undef TRACE_INCLUDE_PATH
32bce29ac9SDaniel Bristot de Oliveira #undef TRACE_INCLUDE_FILE
33bce29ac9SDaniel Bristot de Oliveira #endif /* CONFIG_X86_LOCAL_APIC */
34bce29ac9SDaniel Bristot de Oliveira
35bce29ac9SDaniel Bristot de Oliveira #include <trace/events/irq.h>
36bce29ac9SDaniel Bristot de Oliveira #include <trace/events/sched.h>
37bce29ac9SDaniel Bristot de Oliveira
38bce29ac9SDaniel Bristot de Oliveira #define CREATE_TRACE_POINTS
39bce29ac9SDaniel Bristot de Oliveira #include <trace/events/osnoise.h>
40bce29ac9SDaniel Bristot de Oliveira
41bce29ac9SDaniel Bristot de Oliveira /*
42bce29ac9SDaniel Bristot de Oliveira * Default values.
43bce29ac9SDaniel Bristot de Oliveira */
44bce29ac9SDaniel Bristot de Oliveira #define BANNER "osnoise: "
45bce29ac9SDaniel Bristot de Oliveira #define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */
46bce29ac9SDaniel Bristot de Oliveira #define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */
47bce29ac9SDaniel Bristot de Oliveira
48a955d7eaSDaniel Bristot de Oliveira #define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */
49a955d7eaSDaniel Bristot de Oliveira #define DEFAULT_TIMERLAT_PRIO 95 /* FIFO 95 */
50a955d7eaSDaniel Bristot de Oliveira
51bce29ac9SDaniel Bristot de Oliveira /*
52b179d48bSDaniel Bristot de Oliveira * osnoise/options entries.
53b179d48bSDaniel Bristot de Oliveira */
54b179d48bSDaniel Bristot de Oliveira enum osnoise_options_index {
55b179d48bSDaniel Bristot de Oliveira OSN_DEFAULTS = 0,
5630838fcdSDaniel Bristot de Oliveira OSN_WORKLOAD,
571603dda4SDaniel Bristot de Oliveira OSN_PANIC_ON_STOP,
58b5dce200SDaniel Bristot de Oliveira OSN_PREEMPT_DISABLE,
59b5dce200SDaniel Bristot de Oliveira OSN_IRQ_DISABLE,
60b179d48bSDaniel Bristot de Oliveira OSN_MAX
61b179d48bSDaniel Bristot de Oliveira };
62b179d48bSDaniel Bristot de Oliveira
63b5dce200SDaniel Bristot de Oliveira static const char * const osnoise_options_str[OSN_MAX] = {
64b5dce200SDaniel Bristot de Oliveira "DEFAULTS",
65b5dce200SDaniel Bristot de Oliveira "OSNOISE_WORKLOAD",
66b5dce200SDaniel Bristot de Oliveira "PANIC_ON_STOP",
67b5dce200SDaniel Bristot de Oliveira "OSNOISE_PREEMPT_DISABLE",
68b5dce200SDaniel Bristot de Oliveira "OSNOISE_IRQ_DISABLE" };
69b179d48bSDaniel Bristot de Oliveira
7030838fcdSDaniel Bristot de Oliveira #define OSN_DEFAULT_OPTIONS 0x2
71ec370890SDaniel Bristot de Oliveira static unsigned long osnoise_options = OSN_DEFAULT_OPTIONS;
72b179d48bSDaniel Bristot de Oliveira
73b179d48bSDaniel Bristot de Oliveira /*
74dae18134SDaniel Bristot de Oliveira * trace_array of the enabled osnoise/timerlat instances.
75dae18134SDaniel Bristot de Oliveira */
76dae18134SDaniel Bristot de Oliveira struct osnoise_instance {
77dae18134SDaniel Bristot de Oliveira struct list_head list;
78dae18134SDaniel Bristot de Oliveira struct trace_array *tr;
79dae18134SDaniel Bristot de Oliveira };
80d7458bc0SDaniel Bristot de Oliveira
81d7458bc0SDaniel Bristot de Oliveira static struct list_head osnoise_instances;
82dae18134SDaniel Bristot de Oliveira
osnoise_has_registered_instances(void)83dae18134SDaniel Bristot de Oliveira static bool osnoise_has_registered_instances(void)
84dae18134SDaniel Bristot de Oliveira {
85dae18134SDaniel Bristot de Oliveira return !!list_first_or_null_rcu(&osnoise_instances,
86dae18134SDaniel Bristot de Oliveira struct osnoise_instance,
87dae18134SDaniel Bristot de Oliveira list);
88dae18134SDaniel Bristot de Oliveira }
89dae18134SDaniel Bristot de Oliveira
90dae18134SDaniel Bristot de Oliveira /*
912fac8d64SDaniel Bristot de Oliveira * osnoise_instance_registered - check if a tr is already registered
922fac8d64SDaniel Bristot de Oliveira */
osnoise_instance_registered(struct trace_array * tr)932fac8d64SDaniel Bristot de Oliveira static int osnoise_instance_registered(struct trace_array *tr)
942fac8d64SDaniel Bristot de Oliveira {
952fac8d64SDaniel Bristot de Oliveira struct osnoise_instance *inst;
962fac8d64SDaniel Bristot de Oliveira int found = 0;
972fac8d64SDaniel Bristot de Oliveira
982fac8d64SDaniel Bristot de Oliveira rcu_read_lock();
992fac8d64SDaniel Bristot de Oliveira list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1002fac8d64SDaniel Bristot de Oliveira if (inst->tr == tr)
1012fac8d64SDaniel Bristot de Oliveira found = 1;
1022fac8d64SDaniel Bristot de Oliveira }
1032fac8d64SDaniel Bristot de Oliveira rcu_read_unlock();
1042fac8d64SDaniel Bristot de Oliveira
1052fac8d64SDaniel Bristot de Oliveira return found;
1062fac8d64SDaniel Bristot de Oliveira }
1072fac8d64SDaniel Bristot de Oliveira
1082fac8d64SDaniel Bristot de Oliveira /*
109dae18134SDaniel Bristot de Oliveira * osnoise_register_instance - register a new trace instance
110dae18134SDaniel Bristot de Oliveira *
111dae18134SDaniel Bristot de Oliveira * Register a trace_array *tr in the list of instances running
112dae18134SDaniel Bristot de Oliveira * osnoise/timerlat tracers.
113dae18134SDaniel Bristot de Oliveira */
osnoise_register_instance(struct trace_array * tr)114dae18134SDaniel Bristot de Oliveira static int osnoise_register_instance(struct trace_array *tr)
115dae18134SDaniel Bristot de Oliveira {
116dae18134SDaniel Bristot de Oliveira struct osnoise_instance *inst;
117dae18134SDaniel Bristot de Oliveira
118dae18134SDaniel Bristot de Oliveira /*
119dae18134SDaniel Bristot de Oliveira * register/unregister serialization is provided by trace's
120dae18134SDaniel Bristot de Oliveira * trace_types_lock.
121dae18134SDaniel Bristot de Oliveira */
122dae18134SDaniel Bristot de Oliveira lockdep_assert_held(&trace_types_lock);
123dae18134SDaniel Bristot de Oliveira
124dae18134SDaniel Bristot de Oliveira inst = kmalloc(sizeof(*inst), GFP_KERNEL);
125dae18134SDaniel Bristot de Oliveira if (!inst)
126dae18134SDaniel Bristot de Oliveira return -ENOMEM;
127dae18134SDaniel Bristot de Oliveira
128dae18134SDaniel Bristot de Oliveira INIT_LIST_HEAD_RCU(&inst->list);
129dae18134SDaniel Bristot de Oliveira inst->tr = tr;
130dae18134SDaniel Bristot de Oliveira list_add_tail_rcu(&inst->list, &osnoise_instances);
131dae18134SDaniel Bristot de Oliveira
132dae18134SDaniel Bristot de Oliveira return 0;
133dae18134SDaniel Bristot de Oliveira }
134dae18134SDaniel Bristot de Oliveira
135dae18134SDaniel Bristot de Oliveira /*
136dae18134SDaniel Bristot de Oliveira * osnoise_unregister_instance - unregister a registered trace instance
137dae18134SDaniel Bristot de Oliveira *
138dae18134SDaniel Bristot de Oliveira * Remove the trace_array *tr from the list of instances running
139dae18134SDaniel Bristot de Oliveira * osnoise/timerlat tracers.
140dae18134SDaniel Bristot de Oliveira */
osnoise_unregister_instance(struct trace_array * tr)141dae18134SDaniel Bristot de Oliveira static void osnoise_unregister_instance(struct trace_array *tr)
142dae18134SDaniel Bristot de Oliveira {
143dae18134SDaniel Bristot de Oliveira struct osnoise_instance *inst;
144dae18134SDaniel Bristot de Oliveira int found = 0;
145dae18134SDaniel Bristot de Oliveira
146dae18134SDaniel Bristot de Oliveira /*
147dae18134SDaniel Bristot de Oliveira * register/unregister serialization is provided by trace's
148dae18134SDaniel Bristot de Oliveira * trace_types_lock.
149dae18134SDaniel Bristot de Oliveira */
150685b64e4SChuang Wang list_for_each_entry_rcu(inst, &osnoise_instances, list,
151685b64e4SChuang Wang lockdep_is_held(&trace_types_lock)) {
152dae18134SDaniel Bristot de Oliveira if (inst->tr == tr) {
153dae18134SDaniel Bristot de Oliveira list_del_rcu(&inst->list);
154dae18134SDaniel Bristot de Oliveira found = 1;
155dae18134SDaniel Bristot de Oliveira break;
156dae18134SDaniel Bristot de Oliveira }
157dae18134SDaniel Bristot de Oliveira }
158dae18134SDaniel Bristot de Oliveira
159dae18134SDaniel Bristot de Oliveira if (!found)
160dae18134SDaniel Bristot de Oliveira return;
161dae18134SDaniel Bristot de Oliveira
162cae16f2cSUladzislau Rezki (Sony) kvfree_rcu_mightsleep(inst);
163dae18134SDaniel Bristot de Oliveira }
164dae18134SDaniel Bristot de Oliveira
165dae18134SDaniel Bristot de Oliveira /*
166bce29ac9SDaniel Bristot de Oliveira * NMI runtime info.
167bce29ac9SDaniel Bristot de Oliveira */
168bce29ac9SDaniel Bristot de Oliveira struct osn_nmi {
169bce29ac9SDaniel Bristot de Oliveira u64 count;
170bce29ac9SDaniel Bristot de Oliveira u64 delta_start;
171bce29ac9SDaniel Bristot de Oliveira };
172bce29ac9SDaniel Bristot de Oliveira
173bce29ac9SDaniel Bristot de Oliveira /*
174bce29ac9SDaniel Bristot de Oliveira * IRQ runtime info.
175bce29ac9SDaniel Bristot de Oliveira */
176bce29ac9SDaniel Bristot de Oliveira struct osn_irq {
177bce29ac9SDaniel Bristot de Oliveira u64 count;
178bce29ac9SDaniel Bristot de Oliveira u64 arrival_time;
179bce29ac9SDaniel Bristot de Oliveira u64 delta_start;
180bce29ac9SDaniel Bristot de Oliveira };
181bce29ac9SDaniel Bristot de Oliveira
182a955d7eaSDaniel Bristot de Oliveira #define IRQ_CONTEXT 0
183a955d7eaSDaniel Bristot de Oliveira #define THREAD_CONTEXT 1
184e88ed227SDaniel Bristot de Oliveira #define THREAD_URET 2
185bce29ac9SDaniel Bristot de Oliveira /*
186bce29ac9SDaniel Bristot de Oliveira * sofirq runtime info.
187bce29ac9SDaniel Bristot de Oliveira */
188bce29ac9SDaniel Bristot de Oliveira struct osn_softirq {
189bce29ac9SDaniel Bristot de Oliveira u64 count;
190bce29ac9SDaniel Bristot de Oliveira u64 arrival_time;
191bce29ac9SDaniel Bristot de Oliveira u64 delta_start;
192bce29ac9SDaniel Bristot de Oliveira };
193bce29ac9SDaniel Bristot de Oliveira
194bce29ac9SDaniel Bristot de Oliveira /*
195bce29ac9SDaniel Bristot de Oliveira * thread runtime info.
196bce29ac9SDaniel Bristot de Oliveira */
197bce29ac9SDaniel Bristot de Oliveira struct osn_thread {
198bce29ac9SDaniel Bristot de Oliveira u64 count;
199bce29ac9SDaniel Bristot de Oliveira u64 arrival_time;
200bce29ac9SDaniel Bristot de Oliveira u64 delta_start;
201bce29ac9SDaniel Bristot de Oliveira };
202bce29ac9SDaniel Bristot de Oliveira
203bce29ac9SDaniel Bristot de Oliveira /*
204bce29ac9SDaniel Bristot de Oliveira * Runtime information: this structure saves the runtime information used by
205bce29ac9SDaniel Bristot de Oliveira * one sampling thread.
206bce29ac9SDaniel Bristot de Oliveira */
207bce29ac9SDaniel Bristot de Oliveira struct osnoise_variables {
208bce29ac9SDaniel Bristot de Oliveira struct task_struct *kthread;
209bce29ac9SDaniel Bristot de Oliveira bool sampling;
210bce29ac9SDaniel Bristot de Oliveira pid_t pid;
211bce29ac9SDaniel Bristot de Oliveira struct osn_nmi nmi;
212bce29ac9SDaniel Bristot de Oliveira struct osn_irq irq;
213bce29ac9SDaniel Bristot de Oliveira struct osn_softirq softirq;
214bce29ac9SDaniel Bristot de Oliveira struct osn_thread thread;
215bce29ac9SDaniel Bristot de Oliveira local_t int_counter;
216bce29ac9SDaniel Bristot de Oliveira };
217bce29ac9SDaniel Bristot de Oliveira
218bce29ac9SDaniel Bristot de Oliveira /*
219bce29ac9SDaniel Bristot de Oliveira * Per-cpu runtime information.
220bce29ac9SDaniel Bristot de Oliveira */
2217a025e06STom Rix static DEFINE_PER_CPU(struct osnoise_variables, per_cpu_osnoise_var);
222bce29ac9SDaniel Bristot de Oliveira
223bce29ac9SDaniel Bristot de Oliveira /*
224bce29ac9SDaniel Bristot de Oliveira * this_cpu_osn_var - Return the per-cpu osnoise_variables on its relative CPU
225bce29ac9SDaniel Bristot de Oliveira */
this_cpu_osn_var(void)226bce29ac9SDaniel Bristot de Oliveira static inline struct osnoise_variables *this_cpu_osn_var(void)
227bce29ac9SDaniel Bristot de Oliveira {
228bce29ac9SDaniel Bristot de Oliveira return this_cpu_ptr(&per_cpu_osnoise_var);
229bce29ac9SDaniel Bristot de Oliveira }
230bce29ac9SDaniel Bristot de Oliveira
231a955d7eaSDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
232bce29ac9SDaniel Bristot de Oliveira /*
233a955d7eaSDaniel Bristot de Oliveira * Runtime information for the timer mode.
234bce29ac9SDaniel Bristot de Oliveira */
235a955d7eaSDaniel Bristot de Oliveira struct timerlat_variables {
236a955d7eaSDaniel Bristot de Oliveira struct task_struct *kthread;
237a955d7eaSDaniel Bristot de Oliveira struct hrtimer timer;
238a955d7eaSDaniel Bristot de Oliveira u64 rel_period;
239a955d7eaSDaniel Bristot de Oliveira u64 abs_period;
240a955d7eaSDaniel Bristot de Oliveira bool tracing_thread;
241a955d7eaSDaniel Bristot de Oliveira u64 count;
242e88ed227SDaniel Bristot de Oliveira bool uthread_migrate;
243a955d7eaSDaniel Bristot de Oliveira };
244a955d7eaSDaniel Bristot de Oliveira
2457a025e06STom Rix static DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var);
246a955d7eaSDaniel Bristot de Oliveira
247a955d7eaSDaniel Bristot de Oliveira /*
248a955d7eaSDaniel Bristot de Oliveira * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU
249a955d7eaSDaniel Bristot de Oliveira */
this_cpu_tmr_var(void)250a955d7eaSDaniel Bristot de Oliveira static inline struct timerlat_variables *this_cpu_tmr_var(void)
251bce29ac9SDaniel Bristot de Oliveira {
252a955d7eaSDaniel Bristot de Oliveira return this_cpu_ptr(&per_cpu_timerlat_var);
253a955d7eaSDaniel Bristot de Oliveira }
254a955d7eaSDaniel Bristot de Oliveira
255a955d7eaSDaniel Bristot de Oliveira /*
256a955d7eaSDaniel Bristot de Oliveira * tlat_var_reset - Reset the values of the given timerlat_variables
257a955d7eaSDaniel Bristot de Oliveira */
tlat_var_reset(void)258a955d7eaSDaniel Bristot de Oliveira static inline void tlat_var_reset(void)
259a955d7eaSDaniel Bristot de Oliveira {
260a955d7eaSDaniel Bristot de Oliveira struct timerlat_variables *tlat_var;
261a955d7eaSDaniel Bristot de Oliveira int cpu;
262bce29ac9SDaniel Bristot de Oliveira /*
263bce29ac9SDaniel Bristot de Oliveira * So far, all the values are initialized as 0, so
264bce29ac9SDaniel Bristot de Oliveira * zeroing the structure is perfect.
265bce29ac9SDaniel Bristot de Oliveira */
266a955d7eaSDaniel Bristot de Oliveira for_each_cpu(cpu, cpu_online_mask) {
267a955d7eaSDaniel Bristot de Oliveira tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
268a955d7eaSDaniel Bristot de Oliveira memset(tlat_var, 0, sizeof(*tlat_var));
269a955d7eaSDaniel Bristot de Oliveira }
270a955d7eaSDaniel Bristot de Oliveira }
271a955d7eaSDaniel Bristot de Oliveira #else /* CONFIG_TIMERLAT_TRACER */
272a955d7eaSDaniel Bristot de Oliveira #define tlat_var_reset() do {} while (0)
273a955d7eaSDaniel Bristot de Oliveira #endif /* CONFIG_TIMERLAT_TRACER */
274a955d7eaSDaniel Bristot de Oliveira
275a955d7eaSDaniel Bristot de Oliveira /*
276a955d7eaSDaniel Bristot de Oliveira * osn_var_reset - Reset the values of the given osnoise_variables
277a955d7eaSDaniel Bristot de Oliveira */
osn_var_reset(void)278a955d7eaSDaniel Bristot de Oliveira static inline void osn_var_reset(void)
279a955d7eaSDaniel Bristot de Oliveira {
280a955d7eaSDaniel Bristot de Oliveira struct osnoise_variables *osn_var;
281a955d7eaSDaniel Bristot de Oliveira int cpu;
282a955d7eaSDaniel Bristot de Oliveira
283a955d7eaSDaniel Bristot de Oliveira /*
284a955d7eaSDaniel Bristot de Oliveira * So far, all the values are initialized as 0, so
285a955d7eaSDaniel Bristot de Oliveira * zeroing the structure is perfect.
286a955d7eaSDaniel Bristot de Oliveira */
287a955d7eaSDaniel Bristot de Oliveira for_each_cpu(cpu, cpu_online_mask) {
288a955d7eaSDaniel Bristot de Oliveira osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
289bce29ac9SDaniel Bristot de Oliveira memset(osn_var, 0, sizeof(*osn_var));
290bce29ac9SDaniel Bristot de Oliveira }
291a955d7eaSDaniel Bristot de Oliveira }
292bce29ac9SDaniel Bristot de Oliveira
293bce29ac9SDaniel Bristot de Oliveira /*
294bce29ac9SDaniel Bristot de Oliveira * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables
295bce29ac9SDaniel Bristot de Oliveira */
osn_var_reset_all(void)296bce29ac9SDaniel Bristot de Oliveira static inline void osn_var_reset_all(void)
297bce29ac9SDaniel Bristot de Oliveira {
298a955d7eaSDaniel Bristot de Oliveira osn_var_reset();
299a955d7eaSDaniel Bristot de Oliveira tlat_var_reset();
300bce29ac9SDaniel Bristot de Oliveira }
301bce29ac9SDaniel Bristot de Oliveira
302bce29ac9SDaniel Bristot de Oliveira /*
303bce29ac9SDaniel Bristot de Oliveira * Tells NMIs to call back to the osnoise tracer to record timestamps.
304bce29ac9SDaniel Bristot de Oliveira */
305bce29ac9SDaniel Bristot de Oliveira bool trace_osnoise_callback_enabled;
306bce29ac9SDaniel Bristot de Oliveira
307bce29ac9SDaniel Bristot de Oliveira /*
308bce29ac9SDaniel Bristot de Oliveira * osnoise sample structure definition. Used to store the statistics of a
309bce29ac9SDaniel Bristot de Oliveira * sample run.
310bce29ac9SDaniel Bristot de Oliveira */
311bce29ac9SDaniel Bristot de Oliveira struct osnoise_sample {
312bce29ac9SDaniel Bristot de Oliveira u64 runtime; /* runtime */
313bce29ac9SDaniel Bristot de Oliveira u64 noise; /* noise */
314bce29ac9SDaniel Bristot de Oliveira u64 max_sample; /* max single noise sample */
315bce29ac9SDaniel Bristot de Oliveira int hw_count; /* # HW (incl. hypervisor) interference */
316bce29ac9SDaniel Bristot de Oliveira int nmi_count; /* # NMIs during this sample */
317bce29ac9SDaniel Bristot de Oliveira int irq_count; /* # IRQs during this sample */
318bce29ac9SDaniel Bristot de Oliveira int softirq_count; /* # softirqs during this sample */
319bce29ac9SDaniel Bristot de Oliveira int thread_count; /* # threads during this sample */
320bce29ac9SDaniel Bristot de Oliveira };
321bce29ac9SDaniel Bristot de Oliveira
322a955d7eaSDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
323a955d7eaSDaniel Bristot de Oliveira /*
324a955d7eaSDaniel Bristot de Oliveira * timerlat sample structure definition. Used to store the statistics of
325a955d7eaSDaniel Bristot de Oliveira * a sample run.
326a955d7eaSDaniel Bristot de Oliveira */
327a955d7eaSDaniel Bristot de Oliveira struct timerlat_sample {
328a955d7eaSDaniel Bristot de Oliveira u64 timer_latency; /* timer_latency */
329a955d7eaSDaniel Bristot de Oliveira unsigned int seqnum; /* unique sequence */
330a955d7eaSDaniel Bristot de Oliveira int context; /* timer context */
331a955d7eaSDaniel Bristot de Oliveira };
332a955d7eaSDaniel Bristot de Oliveira #endif
333a955d7eaSDaniel Bristot de Oliveira
334bce29ac9SDaniel Bristot de Oliveira /*
335bce29ac9SDaniel Bristot de Oliveira * Protect the interface.
336bce29ac9SDaniel Bristot de Oliveira */
3377a025e06STom Rix static struct mutex interface_lock;
338bce29ac9SDaniel Bristot de Oliveira
339bce29ac9SDaniel Bristot de Oliveira /*
340bce29ac9SDaniel Bristot de Oliveira * Tracer data.
341bce29ac9SDaniel Bristot de Oliveira */
342bce29ac9SDaniel Bristot de Oliveira static struct osnoise_data {
343bce29ac9SDaniel Bristot de Oliveira u64 sample_period; /* total sampling period */
344bce29ac9SDaniel Bristot de Oliveira u64 sample_runtime; /* active sampling portion of period */
345a955d7eaSDaniel Bristot de Oliveira u64 stop_tracing; /* stop trace in the internal operation (loop/irq) */
346a955d7eaSDaniel Bristot de Oliveira u64 stop_tracing_total; /* stop trace in the final operation (report/thread) */
347a955d7eaSDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
348a955d7eaSDaniel Bristot de Oliveira u64 timerlat_period; /* timerlat period */
349a955d7eaSDaniel Bristot de Oliveira u64 print_stack; /* print IRQ stack if total > */
350a955d7eaSDaniel Bristot de Oliveira int timerlat_tracer; /* timerlat tracer */
351a955d7eaSDaniel Bristot de Oliveira #endif
352bce29ac9SDaniel Bristot de Oliveira bool tainted; /* infor users and developers about a problem */
353bce29ac9SDaniel Bristot de Oliveira } osnoise_data = {
354bce29ac9SDaniel Bristot de Oliveira .sample_period = DEFAULT_SAMPLE_PERIOD,
355bce29ac9SDaniel Bristot de Oliveira .sample_runtime = DEFAULT_SAMPLE_RUNTIME,
356bce29ac9SDaniel Bristot de Oliveira .stop_tracing = 0,
357bce29ac9SDaniel Bristot de Oliveira .stop_tracing_total = 0,
358a955d7eaSDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
359a955d7eaSDaniel Bristot de Oliveira .print_stack = 0,
360a955d7eaSDaniel Bristot de Oliveira .timerlat_period = DEFAULT_TIMERLAT_PERIOD,
361a955d7eaSDaniel Bristot de Oliveira .timerlat_tracer = 0,
362a955d7eaSDaniel Bristot de Oliveira #endif
363bce29ac9SDaniel Bristot de Oliveira };
364bce29ac9SDaniel Bristot de Oliveira
365ccb67544SDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
timerlat_enabled(void)366ccb67544SDaniel Bristot de Oliveira static inline bool timerlat_enabled(void)
367ccb67544SDaniel Bristot de Oliveira {
368ccb67544SDaniel Bristot de Oliveira return osnoise_data.timerlat_tracer;
369ccb67544SDaniel Bristot de Oliveira }
370ccb67544SDaniel Bristot de Oliveira
timerlat_softirq_exit(struct osnoise_variables * osn_var)371ccb67544SDaniel Bristot de Oliveira static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
372ccb67544SDaniel Bristot de Oliveira {
373ccb67544SDaniel Bristot de Oliveira struct timerlat_variables *tlat_var = this_cpu_tmr_var();
374ccb67544SDaniel Bristot de Oliveira /*
375ccb67544SDaniel Bristot de Oliveira * If the timerlat is enabled, but the irq handler did
376ccb67544SDaniel Bristot de Oliveira * not run yet enabling timerlat_tracer, do not trace.
377ccb67544SDaniel Bristot de Oliveira */
378ccb67544SDaniel Bristot de Oliveira if (!tlat_var->tracing_thread) {
379ccb67544SDaniel Bristot de Oliveira osn_var->softirq.arrival_time = 0;
380ccb67544SDaniel Bristot de Oliveira osn_var->softirq.delta_start = 0;
381ccb67544SDaniel Bristot de Oliveira return 0;
382ccb67544SDaniel Bristot de Oliveira }
383ccb67544SDaniel Bristot de Oliveira return 1;
384ccb67544SDaniel Bristot de Oliveira }
385ccb67544SDaniel Bristot de Oliveira
timerlat_thread_exit(struct osnoise_variables * osn_var)386ccb67544SDaniel Bristot de Oliveira static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
387ccb67544SDaniel Bristot de Oliveira {
388ccb67544SDaniel Bristot de Oliveira struct timerlat_variables *tlat_var = this_cpu_tmr_var();
389ccb67544SDaniel Bristot de Oliveira /*
390ccb67544SDaniel Bristot de Oliveira * If the timerlat is enabled, but the irq handler did
391ccb67544SDaniel Bristot de Oliveira * not run yet enabling timerlat_tracer, do not trace.
392ccb67544SDaniel Bristot de Oliveira */
393ccb67544SDaniel Bristot de Oliveira if (!tlat_var->tracing_thread) {
394ccb67544SDaniel Bristot de Oliveira osn_var->thread.delta_start = 0;
395ccb67544SDaniel Bristot de Oliveira osn_var->thread.arrival_time = 0;
396ccb67544SDaniel Bristot de Oliveira return 0;
397ccb67544SDaniel Bristot de Oliveira }
398ccb67544SDaniel Bristot de Oliveira return 1;
399ccb67544SDaniel Bristot de Oliveira }
400ccb67544SDaniel Bristot de Oliveira #else /* CONFIG_TIMERLAT_TRACER */
timerlat_enabled(void)401ccb67544SDaniel Bristot de Oliveira static inline bool timerlat_enabled(void)
402ccb67544SDaniel Bristot de Oliveira {
403ccb67544SDaniel Bristot de Oliveira return false;
404ccb67544SDaniel Bristot de Oliveira }
405ccb67544SDaniel Bristot de Oliveira
timerlat_softirq_exit(struct osnoise_variables * osn_var)406ccb67544SDaniel Bristot de Oliveira static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)
407ccb67544SDaniel Bristot de Oliveira {
408ccb67544SDaniel Bristot de Oliveira return 1;
409ccb67544SDaniel Bristot de Oliveira }
timerlat_thread_exit(struct osnoise_variables * osn_var)410ccb67544SDaniel Bristot de Oliveira static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)
411ccb67544SDaniel Bristot de Oliveira {
412ccb67544SDaniel Bristot de Oliveira return 1;
413ccb67544SDaniel Bristot de Oliveira }
414ccb67544SDaniel Bristot de Oliveira #endif
415ccb67544SDaniel Bristot de Oliveira
416d03721a6SDaniel Bristot de Oliveira #ifdef CONFIG_PREEMPT_RT
417bce29ac9SDaniel Bristot de Oliveira /*
418bce29ac9SDaniel Bristot de Oliveira * Print the osnoise header info.
419bce29ac9SDaniel Bristot de Oliveira */
print_osnoise_headers(struct seq_file * s)420bce29ac9SDaniel Bristot de Oliveira static void print_osnoise_headers(struct seq_file *s)
421bce29ac9SDaniel Bristot de Oliveira {
422bce29ac9SDaniel Bristot de Oliveira if (osnoise_data.tainted)
423bce29ac9SDaniel Bristot de Oliveira seq_puts(s, "# osnoise is tainted!\n");
424bce29ac9SDaniel Bristot de Oliveira
425d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# _-------=> irqs-off\n");
426d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# / _------=> need-resched\n");
427d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# | / _-----=> need-resched-lazy\n");
428d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# || / _----=> hardirq/softirq\n");
429d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# ||| / _---=> preempt-depth\n");
430d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
431d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# ||||| / _-=> migrate-disable\n");
432d03721a6SDaniel Bristot de Oliveira
433d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# |||||| / ");
434d03721a6SDaniel Bristot de Oliveira seq_puts(s, " MAX\n");
435d03721a6SDaniel Bristot de Oliveira
436d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# ||||| / ");
437d03721a6SDaniel Bristot de Oliveira seq_puts(s, " SINGLE Interference counters:\n");
438d03721a6SDaniel Bristot de Oliveira
439d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# ||||||| RUNTIME ");
440d03721a6SDaniel Bristot de Oliveira seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
441d03721a6SDaniel Bristot de Oliveira
442d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP IN US ");
443d03721a6SDaniel Bristot de Oliveira seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
444d03721a6SDaniel Bristot de Oliveira
445d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# | | | ||||||| | | ");
446d03721a6SDaniel Bristot de Oliveira seq_puts(s, " | | | | | | | |\n");
447d03721a6SDaniel Bristot de Oliveira }
448d03721a6SDaniel Bristot de Oliveira #else /* CONFIG_PREEMPT_RT */
print_osnoise_headers(struct seq_file * s)449d03721a6SDaniel Bristot de Oliveira static void print_osnoise_headers(struct seq_file *s)
450d03721a6SDaniel Bristot de Oliveira {
451d03721a6SDaniel Bristot de Oliveira if (osnoise_data.tainted)
452d03721a6SDaniel Bristot de Oliveira seq_puts(s, "# osnoise is tainted!\n");
453d03721a6SDaniel Bristot de Oliveira
454bce29ac9SDaniel Bristot de Oliveira seq_puts(s, "# _-----=> irqs-off\n");
455bce29ac9SDaniel Bristot de Oliveira seq_puts(s, "# / _----=> need-resched\n");
456bce29ac9SDaniel Bristot de Oliveira seq_puts(s, "# | / _---=> hardirq/softirq\n");
457e0f3b18bSDaniel Bristot de Oliveira seq_puts(s, "# || / _--=> preempt-depth\n");
458e0f3b18bSDaniel Bristot de Oliveira seq_puts(s, "# ||| / _-=> migrate-disable ");
459bce29ac9SDaniel Bristot de Oliveira seq_puts(s, " MAX\n");
460e0f3b18bSDaniel Bristot de Oliveira seq_puts(s, "# |||| / delay ");
461bce29ac9SDaniel Bristot de Oliveira seq_puts(s, " SINGLE Interference counters:\n");
462bce29ac9SDaniel Bristot de Oliveira
463e0f3b18bSDaniel Bristot de Oliveira seq_puts(s, "# ||||| RUNTIME ");
464bce29ac9SDaniel Bristot de Oliveira seq_puts(s, " NOISE %% OF CPU NOISE +-----------------------------+\n");
465bce29ac9SDaniel Bristot de Oliveira
466e0f3b18bSDaniel Bristot de Oliveira seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP IN US ");
467bce29ac9SDaniel Bristot de Oliveira seq_puts(s, " IN US AVAILABLE IN US HW NMI IRQ SIRQ THREAD\n");
468bce29ac9SDaniel Bristot de Oliveira
469e0f3b18bSDaniel Bristot de Oliveira seq_puts(s, "# | | | ||||| | | ");
470bce29ac9SDaniel Bristot de Oliveira seq_puts(s, " | | | | | | | |\n");
471bce29ac9SDaniel Bristot de Oliveira }
472d03721a6SDaniel Bristot de Oliveira #endif /* CONFIG_PREEMPT_RT */
473bce29ac9SDaniel Bristot de Oliveira
474bce29ac9SDaniel Bristot de Oliveira /*
475bce29ac9SDaniel Bristot de Oliveira * osnoise_taint - report an osnoise error.
476bce29ac9SDaniel Bristot de Oliveira */
477bce29ac9SDaniel Bristot de Oliveira #define osnoise_taint(msg) ({ \
478dae18134SDaniel Bristot de Oliveira struct osnoise_instance *inst; \
479dae18134SDaniel Bristot de Oliveira struct trace_buffer *buffer; \
480bce29ac9SDaniel Bristot de Oliveira \
481dae18134SDaniel Bristot de Oliveira rcu_read_lock(); \
482dae18134SDaniel Bristot de Oliveira list_for_each_entry_rcu(inst, &osnoise_instances, list) { \
483dae18134SDaniel Bristot de Oliveira buffer = inst->tr->array_buffer.buffer; \
484dae18134SDaniel Bristot de Oliveira trace_array_printk_buf(buffer, _THIS_IP_, msg); \
485dae18134SDaniel Bristot de Oliveira } \
486dae18134SDaniel Bristot de Oliveira rcu_read_unlock(); \
487bce29ac9SDaniel Bristot de Oliveira osnoise_data.tainted = true; \
488bce29ac9SDaniel Bristot de Oliveira })
489bce29ac9SDaniel Bristot de Oliveira
490bce29ac9SDaniel Bristot de Oliveira /*
491bce29ac9SDaniel Bristot de Oliveira * Record an osnoise_sample into the tracer buffer.
492bce29ac9SDaniel Bristot de Oliveira */
493dae18134SDaniel Bristot de Oliveira static void
__trace_osnoise_sample(struct osnoise_sample * sample,struct trace_buffer * buffer)494dae18134SDaniel Bristot de Oliveira __trace_osnoise_sample(struct osnoise_sample *sample, struct trace_buffer *buffer)
495bce29ac9SDaniel Bristot de Oliveira {
496bce29ac9SDaniel Bristot de Oliveira struct trace_event_call *call = &event_osnoise;
497bce29ac9SDaniel Bristot de Oliveira struct ring_buffer_event *event;
498bce29ac9SDaniel Bristot de Oliveira struct osnoise_entry *entry;
499bce29ac9SDaniel Bristot de Oliveira
500bce29ac9SDaniel Bristot de Oliveira event = trace_buffer_lock_reserve(buffer, TRACE_OSNOISE, sizeof(*entry),
501bce29ac9SDaniel Bristot de Oliveira tracing_gen_ctx());
502bce29ac9SDaniel Bristot de Oliveira if (!event)
503bce29ac9SDaniel Bristot de Oliveira return;
504bce29ac9SDaniel Bristot de Oliveira entry = ring_buffer_event_data(event);
505bce29ac9SDaniel Bristot de Oliveira entry->runtime = sample->runtime;
506bce29ac9SDaniel Bristot de Oliveira entry->noise = sample->noise;
507bce29ac9SDaniel Bristot de Oliveira entry->max_sample = sample->max_sample;
508bce29ac9SDaniel Bristot de Oliveira entry->hw_count = sample->hw_count;
509bce29ac9SDaniel Bristot de Oliveira entry->nmi_count = sample->nmi_count;
510bce29ac9SDaniel Bristot de Oliveira entry->irq_count = sample->irq_count;
511bce29ac9SDaniel Bristot de Oliveira entry->softirq_count = sample->softirq_count;
512bce29ac9SDaniel Bristot de Oliveira entry->thread_count = sample->thread_count;
513bce29ac9SDaniel Bristot de Oliveira
514bce29ac9SDaniel Bristot de Oliveira if (!call_filter_check_discard(call, entry, buffer, event))
515bce29ac9SDaniel Bristot de Oliveira trace_buffer_unlock_commit_nostack(buffer, event);
516bce29ac9SDaniel Bristot de Oliveira }
517bce29ac9SDaniel Bristot de Oliveira
518dae18134SDaniel Bristot de Oliveira /*
519dae18134SDaniel Bristot de Oliveira * Record an osnoise_sample on all osnoise instances.
520dae18134SDaniel Bristot de Oliveira */
trace_osnoise_sample(struct osnoise_sample * sample)521dae18134SDaniel Bristot de Oliveira static void trace_osnoise_sample(struct osnoise_sample *sample)
522dae18134SDaniel Bristot de Oliveira {
523dae18134SDaniel Bristot de Oliveira struct osnoise_instance *inst;
524dae18134SDaniel Bristot de Oliveira struct trace_buffer *buffer;
525dae18134SDaniel Bristot de Oliveira
526dae18134SDaniel Bristot de Oliveira rcu_read_lock();
527dae18134SDaniel Bristot de Oliveira list_for_each_entry_rcu(inst, &osnoise_instances, list) {
528dae18134SDaniel Bristot de Oliveira buffer = inst->tr->array_buffer.buffer;
529dae18134SDaniel Bristot de Oliveira __trace_osnoise_sample(sample, buffer);
530dae18134SDaniel Bristot de Oliveira }
531dae18134SDaniel Bristot de Oliveira rcu_read_unlock();
532dae18134SDaniel Bristot de Oliveira }
533dae18134SDaniel Bristot de Oliveira
534a955d7eaSDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
535a955d7eaSDaniel Bristot de Oliveira /*
536a955d7eaSDaniel Bristot de Oliveira * Print the timerlat header info.
537a955d7eaSDaniel Bristot de Oliveira */
538e1c4ad4aSDaniel Bristot de Oliveira #ifdef CONFIG_PREEMPT_RT
print_timerlat_headers(struct seq_file * s)539e1c4ad4aSDaniel Bristot de Oliveira static void print_timerlat_headers(struct seq_file *s)
540e1c4ad4aSDaniel Bristot de Oliveira {
541e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, "# _-------=> irqs-off\n");
542e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, "# / _------=> need-resched\n");
543e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, "# | / _-----=> need-resched-lazy\n");
544e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, "# || / _----=> hardirq/softirq\n");
545e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, "# ||| / _---=> preempt-depth\n");
546e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, "# |||| / _--=> preempt-lazy-depth\n");
547e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, "# ||||| / _-=> migrate-disable\n");
548e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, "# |||||| /\n");
549e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, "# ||||||| ACTIVATION\n");
550e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, "# TASK-PID CPU# ||||||| TIMESTAMP ID ");
551e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, " CONTEXT LATENCY\n");
552e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, "# | | | ||||||| | | ");
553e1c4ad4aSDaniel Bristot de Oliveira seq_puts(s, " | |\n");
554e1c4ad4aSDaniel Bristot de Oliveira }
555e1c4ad4aSDaniel Bristot de Oliveira #else /* CONFIG_PREEMPT_RT */
print_timerlat_headers(struct seq_file * s)556a955d7eaSDaniel Bristot de Oliveira static void print_timerlat_headers(struct seq_file *s)
557a955d7eaSDaniel Bristot de Oliveira {
558a955d7eaSDaniel Bristot de Oliveira seq_puts(s, "# _-----=> irqs-off\n");
559a955d7eaSDaniel Bristot de Oliveira seq_puts(s, "# / _----=> need-resched\n");
560a955d7eaSDaniel Bristot de Oliveira seq_puts(s, "# | / _---=> hardirq/softirq\n");
561a955d7eaSDaniel Bristot de Oliveira seq_puts(s, "# || / _--=> preempt-depth\n");
562aeafcb82SDaniel Bristot de Oliveira seq_puts(s, "# ||| / _-=> migrate-disable\n");
563aeafcb82SDaniel Bristot de Oliveira seq_puts(s, "# |||| / delay\n");
564aeafcb82SDaniel Bristot de Oliveira seq_puts(s, "# ||||| ACTIVATION\n");
565aeafcb82SDaniel Bristot de Oliveira seq_puts(s, "# TASK-PID CPU# ||||| TIMESTAMP ID ");
566a955d7eaSDaniel Bristot de Oliveira seq_puts(s, " CONTEXT LATENCY\n");
567aeafcb82SDaniel Bristot de Oliveira seq_puts(s, "# | | | ||||| | | ");
568a955d7eaSDaniel Bristot de Oliveira seq_puts(s, " | |\n");
569a955d7eaSDaniel Bristot de Oliveira }
570e1c4ad4aSDaniel Bristot de Oliveira #endif /* CONFIG_PREEMPT_RT */
571a955d7eaSDaniel Bristot de Oliveira
572dae18134SDaniel Bristot de Oliveira static void
__trace_timerlat_sample(struct timerlat_sample * sample,struct trace_buffer * buffer)573dae18134SDaniel Bristot de Oliveira __trace_timerlat_sample(struct timerlat_sample *sample, struct trace_buffer *buffer)
574a955d7eaSDaniel Bristot de Oliveira {
575a955d7eaSDaniel Bristot de Oliveira struct trace_event_call *call = &event_osnoise;
576a955d7eaSDaniel Bristot de Oliveira struct ring_buffer_event *event;
577a955d7eaSDaniel Bristot de Oliveira struct timerlat_entry *entry;
578a955d7eaSDaniel Bristot de Oliveira
579a955d7eaSDaniel Bristot de Oliveira event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry),
580a955d7eaSDaniel Bristot de Oliveira tracing_gen_ctx());
581a955d7eaSDaniel Bristot de Oliveira if (!event)
582a955d7eaSDaniel Bristot de Oliveira return;
583a955d7eaSDaniel Bristot de Oliveira entry = ring_buffer_event_data(event);
584a955d7eaSDaniel Bristot de Oliveira entry->seqnum = sample->seqnum;
585a955d7eaSDaniel Bristot de Oliveira entry->context = sample->context;
586a955d7eaSDaniel Bristot de Oliveira entry->timer_latency = sample->timer_latency;
587a955d7eaSDaniel Bristot de Oliveira
588a955d7eaSDaniel Bristot de Oliveira if (!call_filter_check_discard(call, entry, buffer, event))
589a955d7eaSDaniel Bristot de Oliveira trace_buffer_unlock_commit_nostack(buffer, event);
590a955d7eaSDaniel Bristot de Oliveira }
591a955d7eaSDaniel Bristot de Oliveira
592dae18134SDaniel Bristot de Oliveira /*
593dae18134SDaniel Bristot de Oliveira * Record an timerlat_sample into the tracer buffer.
594dae18134SDaniel Bristot de Oliveira */
trace_timerlat_sample(struct timerlat_sample * sample)595dae18134SDaniel Bristot de Oliveira static void trace_timerlat_sample(struct timerlat_sample *sample)
596dae18134SDaniel Bristot de Oliveira {
597dae18134SDaniel Bristot de Oliveira struct osnoise_instance *inst;
598dae18134SDaniel Bristot de Oliveira struct trace_buffer *buffer;
599dae18134SDaniel Bristot de Oliveira
600dae18134SDaniel Bristot de Oliveira rcu_read_lock();
601dae18134SDaniel Bristot de Oliveira list_for_each_entry_rcu(inst, &osnoise_instances, list) {
602dae18134SDaniel Bristot de Oliveira buffer = inst->tr->array_buffer.buffer;
603dae18134SDaniel Bristot de Oliveira __trace_timerlat_sample(sample, buffer);
604dae18134SDaniel Bristot de Oliveira }
605dae18134SDaniel Bristot de Oliveira rcu_read_unlock();
606dae18134SDaniel Bristot de Oliveira }
607dae18134SDaniel Bristot de Oliveira
608a955d7eaSDaniel Bristot de Oliveira #ifdef CONFIG_STACKTRACE
609a955d7eaSDaniel Bristot de Oliveira
610a955d7eaSDaniel Bristot de Oliveira #define MAX_CALLS 256
611a955d7eaSDaniel Bristot de Oliveira
612a955d7eaSDaniel Bristot de Oliveira /*
613a955d7eaSDaniel Bristot de Oliveira * Stack trace will take place only at IRQ level, so, no need
614a955d7eaSDaniel Bristot de Oliveira * to control nesting here.
615a955d7eaSDaniel Bristot de Oliveira */
616a955d7eaSDaniel Bristot de Oliveira struct trace_stack {
617a955d7eaSDaniel Bristot de Oliveira int stack_size;
618a955d7eaSDaniel Bristot de Oliveira int nr_entries;
619a955d7eaSDaniel Bristot de Oliveira unsigned long calls[MAX_CALLS];
620a955d7eaSDaniel Bristot de Oliveira };
621a955d7eaSDaniel Bristot de Oliveira
622a955d7eaSDaniel Bristot de Oliveira static DEFINE_PER_CPU(struct trace_stack, trace_stack);
623a955d7eaSDaniel Bristot de Oliveira
624a955d7eaSDaniel Bristot de Oliveira /*
625a955d7eaSDaniel Bristot de Oliveira * timerlat_save_stack - save a stack trace without printing
626a955d7eaSDaniel Bristot de Oliveira *
627a955d7eaSDaniel Bristot de Oliveira * Save the current stack trace without printing. The
628a955d7eaSDaniel Bristot de Oliveira * stack will be printed later, after the end of the measurement.
629a955d7eaSDaniel Bristot de Oliveira */
timerlat_save_stack(int skip)630a955d7eaSDaniel Bristot de Oliveira static void timerlat_save_stack(int skip)
631a955d7eaSDaniel Bristot de Oliveira {
632a955d7eaSDaniel Bristot de Oliveira unsigned int size, nr_entries;
633a955d7eaSDaniel Bristot de Oliveira struct trace_stack *fstack;
634a955d7eaSDaniel Bristot de Oliveira
635a955d7eaSDaniel Bristot de Oliveira fstack = this_cpu_ptr(&trace_stack);
636a955d7eaSDaniel Bristot de Oliveira
637a955d7eaSDaniel Bristot de Oliveira size = ARRAY_SIZE(fstack->calls);
638a955d7eaSDaniel Bristot de Oliveira
639a955d7eaSDaniel Bristot de Oliveira nr_entries = stack_trace_save(fstack->calls, size, skip);
640a955d7eaSDaniel Bristot de Oliveira
641a955d7eaSDaniel Bristot de Oliveira fstack->stack_size = nr_entries * sizeof(unsigned long);
642a955d7eaSDaniel Bristot de Oliveira fstack->nr_entries = nr_entries;
643a955d7eaSDaniel Bristot de Oliveira
644a955d7eaSDaniel Bristot de Oliveira return;
645a955d7eaSDaniel Bristot de Oliveira
646a955d7eaSDaniel Bristot de Oliveira }
647dae18134SDaniel Bristot de Oliveira
648dae18134SDaniel Bristot de Oliveira static void
__timerlat_dump_stack(struct trace_buffer * buffer,struct trace_stack * fstack,unsigned int size)649dae18134SDaniel Bristot de Oliveira __timerlat_dump_stack(struct trace_buffer *buffer, struct trace_stack *fstack, unsigned int size)
650a955d7eaSDaniel Bristot de Oliveira {
651a955d7eaSDaniel Bristot de Oliveira struct trace_event_call *call = &event_osnoise;
652a955d7eaSDaniel Bristot de Oliveira struct ring_buffer_event *event;
653a955d7eaSDaniel Bristot de Oliveira struct stack_entry *entry;
654a955d7eaSDaniel Bristot de Oliveira
655a955d7eaSDaniel Bristot de Oliveira event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size,
656a955d7eaSDaniel Bristot de Oliveira tracing_gen_ctx());
657a955d7eaSDaniel Bristot de Oliveira if (!event)
658dae18134SDaniel Bristot de Oliveira return;
659a955d7eaSDaniel Bristot de Oliveira
660a955d7eaSDaniel Bristot de Oliveira entry = ring_buffer_event_data(event);
661a955d7eaSDaniel Bristot de Oliveira
662a955d7eaSDaniel Bristot de Oliveira memcpy(&entry->caller, fstack->calls, size);
663a955d7eaSDaniel Bristot de Oliveira entry->size = fstack->nr_entries;
664a955d7eaSDaniel Bristot de Oliveira
665a955d7eaSDaniel Bristot de Oliveira if (!call_filter_check_discard(call, entry, buffer, event))
666a955d7eaSDaniel Bristot de Oliveira trace_buffer_unlock_commit_nostack(buffer, event);
667dae18134SDaniel Bristot de Oliveira }
668a955d7eaSDaniel Bristot de Oliveira
669dae18134SDaniel Bristot de Oliveira /*
670dae18134SDaniel Bristot de Oliveira * timerlat_dump_stack - dump a stack trace previously saved
671dae18134SDaniel Bristot de Oliveira */
timerlat_dump_stack(u64 latency)672b14f4568SDaniel Bristot de Oliveira static void timerlat_dump_stack(u64 latency)
673dae18134SDaniel Bristot de Oliveira {
674dae18134SDaniel Bristot de Oliveira struct osnoise_instance *inst;
675dae18134SDaniel Bristot de Oliveira struct trace_buffer *buffer;
676dae18134SDaniel Bristot de Oliveira struct trace_stack *fstack;
677dae18134SDaniel Bristot de Oliveira unsigned int size;
678dae18134SDaniel Bristot de Oliveira
679b14f4568SDaniel Bristot de Oliveira /*
680b14f4568SDaniel Bristot de Oliveira * trace only if latency > print_stack config, if enabled.
681b14f4568SDaniel Bristot de Oliveira */
682b14f4568SDaniel Bristot de Oliveira if (!osnoise_data.print_stack || osnoise_data.print_stack > latency)
683b14f4568SDaniel Bristot de Oliveira return;
684b14f4568SDaniel Bristot de Oliveira
685dae18134SDaniel Bristot de Oliveira preempt_disable_notrace();
686dae18134SDaniel Bristot de Oliveira fstack = this_cpu_ptr(&trace_stack);
687dae18134SDaniel Bristot de Oliveira size = fstack->stack_size;
688dae18134SDaniel Bristot de Oliveira
689dae18134SDaniel Bristot de Oliveira rcu_read_lock();
690dae18134SDaniel Bristot de Oliveira list_for_each_entry_rcu(inst, &osnoise_instances, list) {
691dae18134SDaniel Bristot de Oliveira buffer = inst->tr->array_buffer.buffer;
692dae18134SDaniel Bristot de Oliveira __timerlat_dump_stack(buffer, fstack, size);
693dae18134SDaniel Bristot de Oliveira
694dae18134SDaniel Bristot de Oliveira }
695dae18134SDaniel Bristot de Oliveira rcu_read_unlock();
696a955d7eaSDaniel Bristot de Oliveira preempt_enable_notrace();
697a955d7eaSDaniel Bristot de Oliveira }
698b14f4568SDaniel Bristot de Oliveira #else /* CONFIG_STACKTRACE */
699b14f4568SDaniel Bristot de Oliveira #define timerlat_dump_stack(u64 latency) do {} while (0)
700a955d7eaSDaniel Bristot de Oliveira #define timerlat_save_stack(a) do {} while (0)
701a955d7eaSDaniel Bristot de Oliveira #endif /* CONFIG_STACKTRACE */
702a955d7eaSDaniel Bristot de Oliveira #endif /* CONFIG_TIMERLAT_TRACER */
703a955d7eaSDaniel Bristot de Oliveira
704bce29ac9SDaniel Bristot de Oliveira /*
705bce29ac9SDaniel Bristot de Oliveira * Macros to encapsulate the time capturing infrastructure.
706bce29ac9SDaniel Bristot de Oliveira */
707bce29ac9SDaniel Bristot de Oliveira #define time_get() trace_clock_local()
708bce29ac9SDaniel Bristot de Oliveira #define time_to_us(x) div_u64(x, 1000)
709bce29ac9SDaniel Bristot de Oliveira #define time_sub(a, b) ((a) - (b))
710bce29ac9SDaniel Bristot de Oliveira
711bce29ac9SDaniel Bristot de Oliveira /*
712bce29ac9SDaniel Bristot de Oliveira * cond_move_irq_delta_start - Forward the delta_start of a running IRQ
713bce29ac9SDaniel Bristot de Oliveira *
714bce29ac9SDaniel Bristot de Oliveira * If an IRQ is preempted by an NMI, its delta_start is pushed forward
715bce29ac9SDaniel Bristot de Oliveira * to discount the NMI interference.
716bce29ac9SDaniel Bristot de Oliveira *
717bce29ac9SDaniel Bristot de Oliveira * See get_int_safe_duration().
718bce29ac9SDaniel Bristot de Oliveira */
719bce29ac9SDaniel Bristot de Oliveira static inline void
cond_move_irq_delta_start(struct osnoise_variables * osn_var,u64 duration)720bce29ac9SDaniel Bristot de Oliveira cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)
721bce29ac9SDaniel Bristot de Oliveira {
722bce29ac9SDaniel Bristot de Oliveira if (osn_var->irq.delta_start)
723bce29ac9SDaniel Bristot de Oliveira osn_var->irq.delta_start += duration;
724bce29ac9SDaniel Bristot de Oliveira }
725bce29ac9SDaniel Bristot de Oliveira
726bce29ac9SDaniel Bristot de Oliveira #ifndef CONFIG_PREEMPT_RT
727bce29ac9SDaniel Bristot de Oliveira /*
728bce29ac9SDaniel Bristot de Oliveira * cond_move_softirq_delta_start - Forward the delta_start of a running softirq.
729bce29ac9SDaniel Bristot de Oliveira *
730bce29ac9SDaniel Bristot de Oliveira * If a softirq is preempted by an IRQ or NMI, its delta_start is pushed
731bce29ac9SDaniel Bristot de Oliveira * forward to discount the interference.
732bce29ac9SDaniel Bristot de Oliveira *
733bce29ac9SDaniel Bristot de Oliveira * See get_int_safe_duration().
734bce29ac9SDaniel Bristot de Oliveira */
735bce29ac9SDaniel Bristot de Oliveira static inline void
cond_move_softirq_delta_start(struct osnoise_variables * osn_var,u64 duration)736bce29ac9SDaniel Bristot de Oliveira cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)
737bce29ac9SDaniel Bristot de Oliveira {
738bce29ac9SDaniel Bristot de Oliveira if (osn_var->softirq.delta_start)
739bce29ac9SDaniel Bristot de Oliveira osn_var->softirq.delta_start += duration;
740bce29ac9SDaniel Bristot de Oliveira }
741bce29ac9SDaniel Bristot de Oliveira #else /* CONFIG_PREEMPT_RT */
742bce29ac9SDaniel Bristot de Oliveira #define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)
743bce29ac9SDaniel Bristot de Oliveira #endif
744bce29ac9SDaniel Bristot de Oliveira
745bce29ac9SDaniel Bristot de Oliveira /*
746bce29ac9SDaniel Bristot de Oliveira * cond_move_thread_delta_start - Forward the delta_start of a running thread
747bce29ac9SDaniel Bristot de Oliveira *
748bce29ac9SDaniel Bristot de Oliveira * If a noisy thread is preempted by an softirq, IRQ or NMI, its delta_start
749bce29ac9SDaniel Bristot de Oliveira * is pushed forward to discount the interference.
750bce29ac9SDaniel Bristot de Oliveira *
751bce29ac9SDaniel Bristot de Oliveira * See get_int_safe_duration().
752bce29ac9SDaniel Bristot de Oliveira */
753bce29ac9SDaniel Bristot de Oliveira static inline void
cond_move_thread_delta_start(struct osnoise_variables * osn_var,u64 duration)754bce29ac9SDaniel Bristot de Oliveira cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)
755bce29ac9SDaniel Bristot de Oliveira {
756bce29ac9SDaniel Bristot de Oliveira if (osn_var->thread.delta_start)
757bce29ac9SDaniel Bristot de Oliveira osn_var->thread.delta_start += duration;
758bce29ac9SDaniel Bristot de Oliveira }
759bce29ac9SDaniel Bristot de Oliveira
760bce29ac9SDaniel Bristot de Oliveira /*
761bce29ac9SDaniel Bristot de Oliveira * get_int_safe_duration - Get the duration of a window
762bce29ac9SDaniel Bristot de Oliveira *
763bce29ac9SDaniel Bristot de Oliveira * The irq, softirq and thread varaibles need to have its duration without
764bce29ac9SDaniel Bristot de Oliveira * the interference from higher priority interrupts. Instead of keeping a
765bce29ac9SDaniel Bristot de Oliveira * variable to discount the interrupt interference from these variables, the
766bce29ac9SDaniel Bristot de Oliveira * starting time of these variables are pushed forward with the interrupt's
767bce29ac9SDaniel Bristot de Oliveira * duration. In this way, a single variable is used to:
768bce29ac9SDaniel Bristot de Oliveira *
769bce29ac9SDaniel Bristot de Oliveira * - Know if a given window is being measured.
770bce29ac9SDaniel Bristot de Oliveira * - Account its duration.
771bce29ac9SDaniel Bristot de Oliveira * - Discount the interference.
772bce29ac9SDaniel Bristot de Oliveira *
773bce29ac9SDaniel Bristot de Oliveira * To avoid getting inconsistent values, e.g.,:
774bce29ac9SDaniel Bristot de Oliveira *
775bce29ac9SDaniel Bristot de Oliveira * now = time_get()
776bce29ac9SDaniel Bristot de Oliveira * ---> interrupt!
777bce29ac9SDaniel Bristot de Oliveira * delta_start -= int duration;
778bce29ac9SDaniel Bristot de Oliveira * <---
779bce29ac9SDaniel Bristot de Oliveira * duration = now - delta_start;
780bce29ac9SDaniel Bristot de Oliveira *
781bce29ac9SDaniel Bristot de Oliveira * result: negative duration if the variable duration before the
782bce29ac9SDaniel Bristot de Oliveira * interrupt was smaller than the interrupt execution.
783bce29ac9SDaniel Bristot de Oliveira *
784bce29ac9SDaniel Bristot de Oliveira * A counter of interrupts is used. If the counter increased, try
785bce29ac9SDaniel Bristot de Oliveira * to capture an interference safe duration.
786bce29ac9SDaniel Bristot de Oliveira */
787bce29ac9SDaniel Bristot de Oliveira static inline s64
get_int_safe_duration(struct osnoise_variables * osn_var,u64 * delta_start)788bce29ac9SDaniel Bristot de Oliveira get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)
789bce29ac9SDaniel Bristot de Oliveira {
790bce29ac9SDaniel Bristot de Oliveira u64 int_counter, now;
791bce29ac9SDaniel Bristot de Oliveira s64 duration;
792bce29ac9SDaniel Bristot de Oliveira
793bce29ac9SDaniel Bristot de Oliveira do {
794bce29ac9SDaniel Bristot de Oliveira int_counter = local_read(&osn_var->int_counter);
795bce29ac9SDaniel Bristot de Oliveira /* synchronize with interrupts */
796bce29ac9SDaniel Bristot de Oliveira barrier();
797bce29ac9SDaniel Bristot de Oliveira
798bce29ac9SDaniel Bristot de Oliveira now = time_get();
799bce29ac9SDaniel Bristot de Oliveira duration = (now - *delta_start);
800bce29ac9SDaniel Bristot de Oliveira
801bce29ac9SDaniel Bristot de Oliveira /* synchronize with interrupts */
802bce29ac9SDaniel Bristot de Oliveira barrier();
803bce29ac9SDaniel Bristot de Oliveira } while (int_counter != local_read(&osn_var->int_counter));
804bce29ac9SDaniel Bristot de Oliveira
805bce29ac9SDaniel Bristot de Oliveira /*
806bce29ac9SDaniel Bristot de Oliveira * This is an evidence of race conditions that cause
807bce29ac9SDaniel Bristot de Oliveira * a value to be "discounted" too much.
808bce29ac9SDaniel Bristot de Oliveira */
809bce29ac9SDaniel Bristot de Oliveira if (duration < 0)
810bce29ac9SDaniel Bristot de Oliveira osnoise_taint("Negative duration!\n");
811bce29ac9SDaniel Bristot de Oliveira
812bce29ac9SDaniel Bristot de Oliveira *delta_start = 0;
813bce29ac9SDaniel Bristot de Oliveira
814bce29ac9SDaniel Bristot de Oliveira return duration;
815bce29ac9SDaniel Bristot de Oliveira }
816bce29ac9SDaniel Bristot de Oliveira
817bce29ac9SDaniel Bristot de Oliveira /*
818bce29ac9SDaniel Bristot de Oliveira *
819bce29ac9SDaniel Bristot de Oliveira * set_int_safe_time - Save the current time on *time, aware of interference
820bce29ac9SDaniel Bristot de Oliveira *
821bce29ac9SDaniel Bristot de Oliveira * Get the time, taking into consideration a possible interference from
822bce29ac9SDaniel Bristot de Oliveira * higher priority interrupts.
823bce29ac9SDaniel Bristot de Oliveira *
824bce29ac9SDaniel Bristot de Oliveira * See get_int_safe_duration() for an explanation.
825bce29ac9SDaniel Bristot de Oliveira */
826bce29ac9SDaniel Bristot de Oliveira static u64
set_int_safe_time(struct osnoise_variables * osn_var,u64 * time)827bce29ac9SDaniel Bristot de Oliveira set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)
828bce29ac9SDaniel Bristot de Oliveira {
829bce29ac9SDaniel Bristot de Oliveira u64 int_counter;
830bce29ac9SDaniel Bristot de Oliveira
831bce29ac9SDaniel Bristot de Oliveira do {
832bce29ac9SDaniel Bristot de Oliveira int_counter = local_read(&osn_var->int_counter);
833bce29ac9SDaniel Bristot de Oliveira /* synchronize with interrupts */
834bce29ac9SDaniel Bristot de Oliveira barrier();
835bce29ac9SDaniel Bristot de Oliveira
836bce29ac9SDaniel Bristot de Oliveira *time = time_get();
837bce29ac9SDaniel Bristot de Oliveira
838bce29ac9SDaniel Bristot de Oliveira /* synchronize with interrupts */
839bce29ac9SDaniel Bristot de Oliveira barrier();
840bce29ac9SDaniel Bristot de Oliveira } while (int_counter != local_read(&osn_var->int_counter));
841bce29ac9SDaniel Bristot de Oliveira
842bce29ac9SDaniel Bristot de Oliveira return int_counter;
843bce29ac9SDaniel Bristot de Oliveira }
844bce29ac9SDaniel Bristot de Oliveira
845a955d7eaSDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
846a955d7eaSDaniel Bristot de Oliveira /*
847a955d7eaSDaniel Bristot de Oliveira * copy_int_safe_time - Copy *src into *desc aware of interference
848a955d7eaSDaniel Bristot de Oliveira */
849a955d7eaSDaniel Bristot de Oliveira static u64
copy_int_safe_time(struct osnoise_variables * osn_var,u64 * dst,u64 * src)850a955d7eaSDaniel Bristot de Oliveira copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)
851a955d7eaSDaniel Bristot de Oliveira {
852a955d7eaSDaniel Bristot de Oliveira u64 int_counter;
853a955d7eaSDaniel Bristot de Oliveira
854a955d7eaSDaniel Bristot de Oliveira do {
855a955d7eaSDaniel Bristot de Oliveira int_counter = local_read(&osn_var->int_counter);
856a955d7eaSDaniel Bristot de Oliveira /* synchronize with interrupts */
857a955d7eaSDaniel Bristot de Oliveira barrier();
858a955d7eaSDaniel Bristot de Oliveira
859a955d7eaSDaniel Bristot de Oliveira *dst = *src;
860a955d7eaSDaniel Bristot de Oliveira
861a955d7eaSDaniel Bristot de Oliveira /* synchronize with interrupts */
862a955d7eaSDaniel Bristot de Oliveira barrier();
863a955d7eaSDaniel Bristot de Oliveira } while (int_counter != local_read(&osn_var->int_counter));
864a955d7eaSDaniel Bristot de Oliveira
865a955d7eaSDaniel Bristot de Oliveira return int_counter;
866a955d7eaSDaniel Bristot de Oliveira }
867a955d7eaSDaniel Bristot de Oliveira #endif /* CONFIG_TIMERLAT_TRACER */
868a955d7eaSDaniel Bristot de Oliveira
869bce29ac9SDaniel Bristot de Oliveira /*
870bce29ac9SDaniel Bristot de Oliveira * trace_osnoise_callback - NMI entry/exit callback
871bce29ac9SDaniel Bristot de Oliveira *
872bce29ac9SDaniel Bristot de Oliveira * This function is called at the entry and exit NMI code. The bool enter
873bce29ac9SDaniel Bristot de Oliveira * distinguishes between either case. This function is used to note a NMI
874bce29ac9SDaniel Bristot de Oliveira * occurrence, compute the noise caused by the NMI, and to remove the noise
875bce29ac9SDaniel Bristot de Oliveira * it is potentially causing on other interference variables.
876bce29ac9SDaniel Bristot de Oliveira */
trace_osnoise_callback(bool enter)877bce29ac9SDaniel Bristot de Oliveira void trace_osnoise_callback(bool enter)
878bce29ac9SDaniel Bristot de Oliveira {
879bce29ac9SDaniel Bristot de Oliveira struct osnoise_variables *osn_var = this_cpu_osn_var();
880bce29ac9SDaniel Bristot de Oliveira u64 duration;
881bce29ac9SDaniel Bristot de Oliveira
882bce29ac9SDaniel Bristot de Oliveira if (!osn_var->sampling)
883bce29ac9SDaniel Bristot de Oliveira return;
884bce29ac9SDaniel Bristot de Oliveira
885bce29ac9SDaniel Bristot de Oliveira /*
886bce29ac9SDaniel Bristot de Oliveira * Currently trace_clock_local() calls sched_clock() and the
887bce29ac9SDaniel Bristot de Oliveira * generic version is not NMI safe.
888bce29ac9SDaniel Bristot de Oliveira */
889bce29ac9SDaniel Bristot de Oliveira if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK)) {
890bce29ac9SDaniel Bristot de Oliveira if (enter) {
891bce29ac9SDaniel Bristot de Oliveira osn_var->nmi.delta_start = time_get();
892bce29ac9SDaniel Bristot de Oliveira local_inc(&osn_var->int_counter);
893bce29ac9SDaniel Bristot de Oliveira } else {
894bce29ac9SDaniel Bristot de Oliveira duration = time_get() - osn_var->nmi.delta_start;
895bce29ac9SDaniel Bristot de Oliveira
896bce29ac9SDaniel Bristot de Oliveira trace_nmi_noise(osn_var->nmi.delta_start, duration);
897bce29ac9SDaniel Bristot de Oliveira
898bce29ac9SDaniel Bristot de Oliveira cond_move_irq_delta_start(osn_var, duration);
899bce29ac9SDaniel Bristot de Oliveira cond_move_softirq_delta_start(osn_var, duration);
900bce29ac9SDaniel Bristot de Oliveira cond_move_thread_delta_start(osn_var, duration);
901bce29ac9SDaniel Bristot de Oliveira }
902bce29ac9SDaniel Bristot de Oliveira }
903bce29ac9SDaniel Bristot de Oliveira
904bce29ac9SDaniel Bristot de Oliveira if (enter)
905bce29ac9SDaniel Bristot de Oliveira osn_var->nmi.count++;
906bce29ac9SDaniel Bristot de Oliveira }
907bce29ac9SDaniel Bristot de Oliveira
908bce29ac9SDaniel Bristot de Oliveira /*
909bce29ac9SDaniel Bristot de Oliveira * osnoise_trace_irq_entry - Note the starting of an IRQ
910bce29ac9SDaniel Bristot de Oliveira *
911bce29ac9SDaniel Bristot de Oliveira * Save the starting time of an IRQ. As IRQs are non-preemptive to other IRQs,
912bce29ac9SDaniel Bristot de Oliveira * it is safe to use a single variable (ons_var->irq) to save the statistics.
913bce29ac9SDaniel Bristot de Oliveira * The arrival_time is used to report... the arrival time. The delta_start
914bce29ac9SDaniel Bristot de Oliveira * is used to compute the duration at the IRQ exit handler. See
915bce29ac9SDaniel Bristot de Oliveira * cond_move_irq_delta_start().
916bce29ac9SDaniel Bristot de Oliveira */
osnoise_trace_irq_entry(int id)917bce29ac9SDaniel Bristot de Oliveira void osnoise_trace_irq_entry(int id)
918bce29ac9SDaniel Bristot de Oliveira {
919bce29ac9SDaniel Bristot de Oliveira struct osnoise_variables *osn_var = this_cpu_osn_var();
920bce29ac9SDaniel Bristot de Oliveira
921bce29ac9SDaniel Bristot de Oliveira if (!osn_var->sampling)
922bce29ac9SDaniel Bristot de Oliveira return;
923bce29ac9SDaniel Bristot de Oliveira /*
924bce29ac9SDaniel Bristot de Oliveira * This value will be used in the report, but not to compute
925bce29ac9SDaniel Bristot de Oliveira * the execution time, so it is safe to get it unsafe.
926bce29ac9SDaniel Bristot de Oliveira */
927bce29ac9SDaniel Bristot de Oliveira osn_var->irq.arrival_time = time_get();
928bce29ac9SDaniel Bristot de Oliveira set_int_safe_time(osn_var, &osn_var->irq.delta_start);
929bce29ac9SDaniel Bristot de Oliveira osn_var->irq.count++;
930bce29ac9SDaniel Bristot de Oliveira
931bce29ac9SDaniel Bristot de Oliveira local_inc(&osn_var->int_counter);
932bce29ac9SDaniel Bristot de Oliveira }
933bce29ac9SDaniel Bristot de Oliveira
934bce29ac9SDaniel Bristot de Oliveira /*
935bce29ac9SDaniel Bristot de Oliveira * osnoise_irq_exit - Note the end of an IRQ, sava data and trace
936bce29ac9SDaniel Bristot de Oliveira *
937bce29ac9SDaniel Bristot de Oliveira * Computes the duration of the IRQ noise, and trace it. Also discounts the
938bce29ac9SDaniel Bristot de Oliveira * interference from other sources of noise could be currently being accounted.
939bce29ac9SDaniel Bristot de Oliveira */
osnoise_trace_irq_exit(int id,const char * desc)940bce29ac9SDaniel Bristot de Oliveira void osnoise_trace_irq_exit(int id, const char *desc)
941bce29ac9SDaniel Bristot de Oliveira {
942bce29ac9SDaniel Bristot de Oliveira struct osnoise_variables *osn_var = this_cpu_osn_var();
943022632f6SDaniel Bristot de Oliveira s64 duration;
944bce29ac9SDaniel Bristot de Oliveira
945bce29ac9SDaniel Bristot de Oliveira if (!osn_var->sampling)
946bce29ac9SDaniel Bristot de Oliveira return;
947bce29ac9SDaniel Bristot de Oliveira
948bce29ac9SDaniel Bristot de Oliveira duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);
949bce29ac9SDaniel Bristot de Oliveira trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);
950bce29ac9SDaniel Bristot de Oliveira osn_var->irq.arrival_time = 0;
951bce29ac9SDaniel Bristot de Oliveira cond_move_softirq_delta_start(osn_var, duration);
952bce29ac9SDaniel Bristot de Oliveira cond_move_thread_delta_start(osn_var, duration);
953bce29ac9SDaniel Bristot de Oliveira }
954bce29ac9SDaniel Bristot de Oliveira
955bce29ac9SDaniel Bristot de Oliveira /*
956bce29ac9SDaniel Bristot de Oliveira * trace_irqentry_callback - Callback to the irq:irq_entry traceevent
957bce29ac9SDaniel Bristot de Oliveira *
958bce29ac9SDaniel Bristot de Oliveira * Used to note the starting of an IRQ occurece.
959bce29ac9SDaniel Bristot de Oliveira */
trace_irqentry_callback(void * data,int irq,struct irqaction * action)960bce29ac9SDaniel Bristot de Oliveira static void trace_irqentry_callback(void *data, int irq,
961bce29ac9SDaniel Bristot de Oliveira struct irqaction *action)
962bce29ac9SDaniel Bristot de Oliveira {
963bce29ac9SDaniel Bristot de Oliveira osnoise_trace_irq_entry(irq);
964bce29ac9SDaniel Bristot de Oliveira }
965bce29ac9SDaniel Bristot de Oliveira
966bce29ac9SDaniel Bristot de Oliveira /*
967bce29ac9SDaniel Bristot de Oliveira * trace_irqexit_callback - Callback to the irq:irq_exit traceevent
968bce29ac9SDaniel Bristot de Oliveira *
969bce29ac9SDaniel Bristot de Oliveira * Used to note the end of an IRQ occurece.
970bce29ac9SDaniel Bristot de Oliveira */
trace_irqexit_callback(void * data,int irq,struct irqaction * action,int ret)971bce29ac9SDaniel Bristot de Oliveira static void trace_irqexit_callback(void *data, int irq,
972bce29ac9SDaniel Bristot de Oliveira struct irqaction *action, int ret)
973bce29ac9SDaniel Bristot de Oliveira {
974bce29ac9SDaniel Bristot de Oliveira osnoise_trace_irq_exit(irq, action->name);
975bce29ac9SDaniel Bristot de Oliveira }
976bce29ac9SDaniel Bristot de Oliveira
977bce29ac9SDaniel Bristot de Oliveira /*
978bce29ac9SDaniel Bristot de Oliveira * arch specific register function.
979bce29ac9SDaniel Bristot de Oliveira */
osnoise_arch_register(void)980bce29ac9SDaniel Bristot de Oliveira int __weak osnoise_arch_register(void)
981bce29ac9SDaniel Bristot de Oliveira {
982bce29ac9SDaniel Bristot de Oliveira return 0;
983bce29ac9SDaniel Bristot de Oliveira }
984bce29ac9SDaniel Bristot de Oliveira
985bce29ac9SDaniel Bristot de Oliveira /*
986bce29ac9SDaniel Bristot de Oliveira * arch specific unregister function.
987bce29ac9SDaniel Bristot de Oliveira */
osnoise_arch_unregister(void)988bce29ac9SDaniel Bristot de Oliveira void __weak osnoise_arch_unregister(void)
989bce29ac9SDaniel Bristot de Oliveira {
990bce29ac9SDaniel Bristot de Oliveira return;
991bce29ac9SDaniel Bristot de Oliveira }
992bce29ac9SDaniel Bristot de Oliveira
993bce29ac9SDaniel Bristot de Oliveira /*
994bce29ac9SDaniel Bristot de Oliveira * hook_irq_events - Hook IRQ handling events
995bce29ac9SDaniel Bristot de Oliveira *
996bce29ac9SDaniel Bristot de Oliveira * This function hooks the IRQ related callbacks to the respective trace
997bce29ac9SDaniel Bristot de Oliveira * events.
998bce29ac9SDaniel Bristot de Oliveira */
hook_irq_events(void)999f7d9f637SDaniel Bristot de Oliveira static int hook_irq_events(void)
1000bce29ac9SDaniel Bristot de Oliveira {
1001bce29ac9SDaniel Bristot de Oliveira int ret;
1002bce29ac9SDaniel Bristot de Oliveira
1003bce29ac9SDaniel Bristot de Oliveira ret = register_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1004bce29ac9SDaniel Bristot de Oliveira if (ret)
1005bce29ac9SDaniel Bristot de Oliveira goto out_err;
1006bce29ac9SDaniel Bristot de Oliveira
1007bce29ac9SDaniel Bristot de Oliveira ret = register_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1008bce29ac9SDaniel Bristot de Oliveira if (ret)
1009bce29ac9SDaniel Bristot de Oliveira goto out_unregister_entry;
1010bce29ac9SDaniel Bristot de Oliveira
1011bce29ac9SDaniel Bristot de Oliveira ret = osnoise_arch_register();
1012bce29ac9SDaniel Bristot de Oliveira if (ret)
1013bce29ac9SDaniel Bristot de Oliveira goto out_irq_exit;
1014bce29ac9SDaniel Bristot de Oliveira
1015bce29ac9SDaniel Bristot de Oliveira return 0;
1016bce29ac9SDaniel Bristot de Oliveira
1017bce29ac9SDaniel Bristot de Oliveira out_irq_exit:
1018bce29ac9SDaniel Bristot de Oliveira unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1019bce29ac9SDaniel Bristot de Oliveira out_unregister_entry:
1020bce29ac9SDaniel Bristot de Oliveira unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1021bce29ac9SDaniel Bristot de Oliveira out_err:
1022bce29ac9SDaniel Bristot de Oliveira return -EINVAL;
1023bce29ac9SDaniel Bristot de Oliveira }
1024bce29ac9SDaniel Bristot de Oliveira
1025bce29ac9SDaniel Bristot de Oliveira /*
1026bce29ac9SDaniel Bristot de Oliveira * unhook_irq_events - Unhook IRQ handling events
1027bce29ac9SDaniel Bristot de Oliveira *
1028bce29ac9SDaniel Bristot de Oliveira * This function unhooks the IRQ related callbacks to the respective trace
1029bce29ac9SDaniel Bristot de Oliveira * events.
1030bce29ac9SDaniel Bristot de Oliveira */
unhook_irq_events(void)1031f7d9f637SDaniel Bristot de Oliveira static void unhook_irq_events(void)
1032bce29ac9SDaniel Bristot de Oliveira {
1033bce29ac9SDaniel Bristot de Oliveira osnoise_arch_unregister();
1034bce29ac9SDaniel Bristot de Oliveira unregister_trace_irq_handler_exit(trace_irqexit_callback, NULL);
1035bce29ac9SDaniel Bristot de Oliveira unregister_trace_irq_handler_entry(trace_irqentry_callback, NULL);
1036bce29ac9SDaniel Bristot de Oliveira }
1037bce29ac9SDaniel Bristot de Oliveira
1038bce29ac9SDaniel Bristot de Oliveira #ifndef CONFIG_PREEMPT_RT
1039bce29ac9SDaniel Bristot de Oliveira /*
1040bce29ac9SDaniel Bristot de Oliveira * trace_softirq_entry_callback - Note the starting of a softirq
1041bce29ac9SDaniel Bristot de Oliveira *
1042bce29ac9SDaniel Bristot de Oliveira * Save the starting time of a softirq. As softirqs are non-preemptive to
1043bce29ac9SDaniel Bristot de Oliveira * other softirqs, it is safe to use a single variable (ons_var->softirq)
1044bce29ac9SDaniel Bristot de Oliveira * to save the statistics. The arrival_time is used to report... the
1045bce29ac9SDaniel Bristot de Oliveira * arrival time. The delta_start is used to compute the duration at the
1046bce29ac9SDaniel Bristot de Oliveira * softirq exit handler. See cond_move_softirq_delta_start().
1047bce29ac9SDaniel Bristot de Oliveira */
trace_softirq_entry_callback(void * data,unsigned int vec_nr)1048f7d9f637SDaniel Bristot de Oliveira static void trace_softirq_entry_callback(void *data, unsigned int vec_nr)
1049bce29ac9SDaniel Bristot de Oliveira {
1050bce29ac9SDaniel Bristot de Oliveira struct osnoise_variables *osn_var = this_cpu_osn_var();
1051bce29ac9SDaniel Bristot de Oliveira
1052bce29ac9SDaniel Bristot de Oliveira if (!osn_var->sampling)
1053bce29ac9SDaniel Bristot de Oliveira return;
1054bce29ac9SDaniel Bristot de Oliveira /*
1055bce29ac9SDaniel Bristot de Oliveira * This value will be used in the report, but not to compute
1056bce29ac9SDaniel Bristot de Oliveira * the execution time, so it is safe to get it unsafe.
1057bce29ac9SDaniel Bristot de Oliveira */
1058bce29ac9SDaniel Bristot de Oliveira osn_var->softirq.arrival_time = time_get();
1059bce29ac9SDaniel Bristot de Oliveira set_int_safe_time(osn_var, &osn_var->softirq.delta_start);
1060bce29ac9SDaniel Bristot de Oliveira osn_var->softirq.count++;
1061bce29ac9SDaniel Bristot de Oliveira
1062bce29ac9SDaniel Bristot de Oliveira local_inc(&osn_var->int_counter);
1063bce29ac9SDaniel Bristot de Oliveira }
1064bce29ac9SDaniel Bristot de Oliveira
1065bce29ac9SDaniel Bristot de Oliveira /*
1066bce29ac9SDaniel Bristot de Oliveira * trace_softirq_exit_callback - Note the end of an softirq
1067bce29ac9SDaniel Bristot de Oliveira *
1068bce29ac9SDaniel Bristot de Oliveira * Computes the duration of the softirq noise, and trace it. Also discounts the
1069bce29ac9SDaniel Bristot de Oliveira * interference from other sources of noise could be currently being accounted.
1070bce29ac9SDaniel Bristot de Oliveira */
trace_softirq_exit_callback(void * data,unsigned int vec_nr)1071f7d9f637SDaniel Bristot de Oliveira static void trace_softirq_exit_callback(void *data, unsigned int vec_nr)
1072bce29ac9SDaniel Bristot de Oliveira {
1073bce29ac9SDaniel Bristot de Oliveira struct osnoise_variables *osn_var = this_cpu_osn_var();
1074022632f6SDaniel Bristot de Oliveira s64 duration;
1075bce29ac9SDaniel Bristot de Oliveira
1076bce29ac9SDaniel Bristot de Oliveira if (!osn_var->sampling)
1077bce29ac9SDaniel Bristot de Oliveira return;
1078bce29ac9SDaniel Bristot de Oliveira
1079ccb67544SDaniel Bristot de Oliveira if (unlikely(timerlat_enabled()))
1080ccb67544SDaniel Bristot de Oliveira if (!timerlat_softirq_exit(osn_var))
1081a955d7eaSDaniel Bristot de Oliveira return;
1082a955d7eaSDaniel Bristot de Oliveira
1083bce29ac9SDaniel Bristot de Oliveira duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);
1084bce29ac9SDaniel Bristot de Oliveira trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);
1085bce29ac9SDaniel Bristot de Oliveira cond_move_thread_delta_start(osn_var, duration);
1086bce29ac9SDaniel Bristot de Oliveira osn_var->softirq.arrival_time = 0;
1087bce29ac9SDaniel Bristot de Oliveira }
1088bce29ac9SDaniel Bristot de Oliveira
1089bce29ac9SDaniel Bristot de Oliveira /*
1090bce29ac9SDaniel Bristot de Oliveira * hook_softirq_events - Hook softirq handling events
1091bce29ac9SDaniel Bristot de Oliveira *
1092bce29ac9SDaniel Bristot de Oliveira * This function hooks the softirq related callbacks to the respective trace
1093bce29ac9SDaniel Bristot de Oliveira * events.
1094bce29ac9SDaniel Bristot de Oliveira */
hook_softirq_events(void)1095bce29ac9SDaniel Bristot de Oliveira static int hook_softirq_events(void)
1096bce29ac9SDaniel Bristot de Oliveira {
1097bce29ac9SDaniel Bristot de Oliveira int ret;
1098bce29ac9SDaniel Bristot de Oliveira
1099bce29ac9SDaniel Bristot de Oliveira ret = register_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1100bce29ac9SDaniel Bristot de Oliveira if (ret)
1101bce29ac9SDaniel Bristot de Oliveira goto out_err;
1102bce29ac9SDaniel Bristot de Oliveira
1103bce29ac9SDaniel Bristot de Oliveira ret = register_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1104bce29ac9SDaniel Bristot de Oliveira if (ret)
1105bce29ac9SDaniel Bristot de Oliveira goto out_unreg_entry;
1106bce29ac9SDaniel Bristot de Oliveira
1107bce29ac9SDaniel Bristot de Oliveira return 0;
1108bce29ac9SDaniel Bristot de Oliveira
1109bce29ac9SDaniel Bristot de Oliveira out_unreg_entry:
1110bce29ac9SDaniel Bristot de Oliveira unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1111bce29ac9SDaniel Bristot de Oliveira out_err:
1112bce29ac9SDaniel Bristot de Oliveira return -EINVAL;
1113bce29ac9SDaniel Bristot de Oliveira }
1114bce29ac9SDaniel Bristot de Oliveira
1115bce29ac9SDaniel Bristot de Oliveira /*
1116bce29ac9SDaniel Bristot de Oliveira * unhook_softirq_events - Unhook softirq handling events
1117bce29ac9SDaniel Bristot de Oliveira *
1118bce29ac9SDaniel Bristot de Oliveira * This function hooks the softirq related callbacks to the respective trace
1119bce29ac9SDaniel Bristot de Oliveira * events.
1120bce29ac9SDaniel Bristot de Oliveira */
unhook_softirq_events(void)1121bce29ac9SDaniel Bristot de Oliveira static void unhook_softirq_events(void)
1122bce29ac9SDaniel Bristot de Oliveira {
1123bce29ac9SDaniel Bristot de Oliveira unregister_trace_softirq_entry(trace_softirq_entry_callback, NULL);
1124bce29ac9SDaniel Bristot de Oliveira unregister_trace_softirq_exit(trace_softirq_exit_callback, NULL);
1125bce29ac9SDaniel Bristot de Oliveira }
1126bce29ac9SDaniel Bristot de Oliveira #else /* CONFIG_PREEMPT_RT */
1127bce29ac9SDaniel Bristot de Oliveira /*
1128bce29ac9SDaniel Bristot de Oliveira * softirq are threads on the PREEMPT_RT mode.
1129bce29ac9SDaniel Bristot de Oliveira */
hook_softirq_events(void)1130bce29ac9SDaniel Bristot de Oliveira static int hook_softirq_events(void)
1131bce29ac9SDaniel Bristot de Oliveira {
1132bce29ac9SDaniel Bristot de Oliveira return 0;
1133bce29ac9SDaniel Bristot de Oliveira }
unhook_softirq_events(void)1134bce29ac9SDaniel Bristot de Oliveira static void unhook_softirq_events(void)
1135bce29ac9SDaniel Bristot de Oliveira {
1136bce29ac9SDaniel Bristot de Oliveira }
1137bce29ac9SDaniel Bristot de Oliveira #endif
1138bce29ac9SDaniel Bristot de Oliveira
1139bce29ac9SDaniel Bristot de Oliveira /*
1140bce29ac9SDaniel Bristot de Oliveira * thread_entry - Record the starting of a thread noise window
1141bce29ac9SDaniel Bristot de Oliveira *
1142bce29ac9SDaniel Bristot de Oliveira * It saves the context switch time for a noisy thread, and increments
1143bce29ac9SDaniel Bristot de Oliveira * the interference counters.
1144bce29ac9SDaniel Bristot de Oliveira */
1145bce29ac9SDaniel Bristot de Oliveira static void
thread_entry(struct osnoise_variables * osn_var,struct task_struct * t)1146bce29ac9SDaniel Bristot de Oliveira thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)
1147bce29ac9SDaniel Bristot de Oliveira {
1148bce29ac9SDaniel Bristot de Oliveira if (!osn_var->sampling)
1149bce29ac9SDaniel Bristot de Oliveira return;
1150bce29ac9SDaniel Bristot de Oliveira /*
1151bce29ac9SDaniel Bristot de Oliveira * The arrival time will be used in the report, but not to compute
1152bce29ac9SDaniel Bristot de Oliveira * the execution time, so it is safe to get it unsafe.
1153bce29ac9SDaniel Bristot de Oliveira */
1154bce29ac9SDaniel Bristot de Oliveira osn_var->thread.arrival_time = time_get();
1155bce29ac9SDaniel Bristot de Oliveira
1156bce29ac9SDaniel Bristot de Oliveira set_int_safe_time(osn_var, &osn_var->thread.delta_start);
1157bce29ac9SDaniel Bristot de Oliveira
1158bce29ac9SDaniel Bristot de Oliveira osn_var->thread.count++;
1159bce29ac9SDaniel Bristot de Oliveira local_inc(&osn_var->int_counter);
1160bce29ac9SDaniel Bristot de Oliveira }
1161bce29ac9SDaniel Bristot de Oliveira
1162bce29ac9SDaniel Bristot de Oliveira /*
1163bce29ac9SDaniel Bristot de Oliveira * thread_exit - Report the end of a thread noise window
1164bce29ac9SDaniel Bristot de Oliveira *
1165bce29ac9SDaniel Bristot de Oliveira * It computes the total noise from a thread, tracing if needed.
1166bce29ac9SDaniel Bristot de Oliveira */
1167bce29ac9SDaniel Bristot de Oliveira static void
thread_exit(struct osnoise_variables * osn_var,struct task_struct * t)1168bce29ac9SDaniel Bristot de Oliveira thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)
1169bce29ac9SDaniel Bristot de Oliveira {
1170022632f6SDaniel Bristot de Oliveira s64 duration;
1171bce29ac9SDaniel Bristot de Oliveira
1172bce29ac9SDaniel Bristot de Oliveira if (!osn_var->sampling)
1173bce29ac9SDaniel Bristot de Oliveira return;
1174bce29ac9SDaniel Bristot de Oliveira
1175ccb67544SDaniel Bristot de Oliveira if (unlikely(timerlat_enabled()))
1176ccb67544SDaniel Bristot de Oliveira if (!timerlat_thread_exit(osn_var))
1177a955d7eaSDaniel Bristot de Oliveira return;
1178a955d7eaSDaniel Bristot de Oliveira
1179bce29ac9SDaniel Bristot de Oliveira duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);
1180bce29ac9SDaniel Bristot de Oliveira
1181bce29ac9SDaniel Bristot de Oliveira trace_thread_noise(t, osn_var->thread.arrival_time, duration);
1182bce29ac9SDaniel Bristot de Oliveira
1183bce29ac9SDaniel Bristot de Oliveira osn_var->thread.arrival_time = 0;
1184bce29ac9SDaniel Bristot de Oliveira }
1185bce29ac9SDaniel Bristot de Oliveira
1186e88ed227SDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
1187e88ed227SDaniel Bristot de Oliveira /*
1188e88ed227SDaniel Bristot de Oliveira * osnoise_stop_exception - Stop tracing and the tracer.
1189e88ed227SDaniel Bristot de Oliveira */
osnoise_stop_exception(char * msg,int cpu)1190e88ed227SDaniel Bristot de Oliveira static __always_inline void osnoise_stop_exception(char *msg, int cpu)
1191e88ed227SDaniel Bristot de Oliveira {
1192e88ed227SDaniel Bristot de Oliveira struct osnoise_instance *inst;
1193e88ed227SDaniel Bristot de Oliveira struct trace_array *tr;
1194e88ed227SDaniel Bristot de Oliveira
1195e88ed227SDaniel Bristot de Oliveira rcu_read_lock();
1196e88ed227SDaniel Bristot de Oliveira list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1197e88ed227SDaniel Bristot de Oliveira tr = inst->tr;
1198e88ed227SDaniel Bristot de Oliveira trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
1199e88ed227SDaniel Bristot de Oliveira "stop tracing hit on cpu %d due to exception: %s\n",
1200e88ed227SDaniel Bristot de Oliveira smp_processor_id(),
1201e88ed227SDaniel Bristot de Oliveira msg);
1202e88ed227SDaniel Bristot de Oliveira
1203e88ed227SDaniel Bristot de Oliveira if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
1204e88ed227SDaniel Bristot de Oliveira panic("tracer hit on cpu %d due to exception: %s\n",
1205e88ed227SDaniel Bristot de Oliveira smp_processor_id(),
1206e88ed227SDaniel Bristot de Oliveira msg);
1207e88ed227SDaniel Bristot de Oliveira
1208e88ed227SDaniel Bristot de Oliveira tracer_tracing_off(tr);
1209e88ed227SDaniel Bristot de Oliveira }
1210e88ed227SDaniel Bristot de Oliveira rcu_read_unlock();
1211e88ed227SDaniel Bristot de Oliveira }
1212e88ed227SDaniel Bristot de Oliveira
1213e88ed227SDaniel Bristot de Oliveira /*
1214e88ed227SDaniel Bristot de Oliveira * trace_sched_migrate_callback - sched:sched_migrate_task trace event handler
1215e88ed227SDaniel Bristot de Oliveira *
1216e88ed227SDaniel Bristot de Oliveira * his function is hooked to the sched:sched_migrate_task trace event, and monitors
1217e88ed227SDaniel Bristot de Oliveira * timerlat user-space thread migration.
1218e88ed227SDaniel Bristot de Oliveira */
trace_sched_migrate_callback(void * data,struct task_struct * p,int dest_cpu)1219e88ed227SDaniel Bristot de Oliveira static void trace_sched_migrate_callback(void *data, struct task_struct *p, int dest_cpu)
1220e88ed227SDaniel Bristot de Oliveira {
1221e88ed227SDaniel Bristot de Oliveira struct osnoise_variables *osn_var;
1222e88ed227SDaniel Bristot de Oliveira long cpu = task_cpu(p);
1223e88ed227SDaniel Bristot de Oliveira
1224e88ed227SDaniel Bristot de Oliveira osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
1225e88ed227SDaniel Bristot de Oliveira if (osn_var->pid == p->pid && dest_cpu != cpu) {
1226e88ed227SDaniel Bristot de Oliveira per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1;
1227e88ed227SDaniel Bristot de Oliveira osnoise_taint("timerlat user-thread migrated\n");
1228e88ed227SDaniel Bristot de Oliveira osnoise_stop_exception("timerlat user-thread migrated", cpu);
1229e88ed227SDaniel Bristot de Oliveira }
1230e88ed227SDaniel Bristot de Oliveira }
1231e88ed227SDaniel Bristot de Oliveira
register_migration_monitor(void)1232e88ed227SDaniel Bristot de Oliveira static int register_migration_monitor(void)
1233e88ed227SDaniel Bristot de Oliveira {
1234e88ed227SDaniel Bristot de Oliveira int ret = 0;
1235e88ed227SDaniel Bristot de Oliveira
1236e88ed227SDaniel Bristot de Oliveira /*
1237e88ed227SDaniel Bristot de Oliveira * Timerlat thread migration check is only required when running timerlat in user-space.
1238e88ed227SDaniel Bristot de Oliveira * Thus, enable callback only if timerlat is set with no workload.
1239e88ed227SDaniel Bristot de Oliveira */
1240e88ed227SDaniel Bristot de Oliveira if (timerlat_enabled() && !test_bit(OSN_WORKLOAD, &osnoise_options))
1241e88ed227SDaniel Bristot de Oliveira ret = register_trace_sched_migrate_task(trace_sched_migrate_callback, NULL);
1242e88ed227SDaniel Bristot de Oliveira
1243e88ed227SDaniel Bristot de Oliveira return ret;
1244e88ed227SDaniel Bristot de Oliveira }
1245e88ed227SDaniel Bristot de Oliveira
unregister_migration_monitor(void)1246e88ed227SDaniel Bristot de Oliveira static void unregister_migration_monitor(void)
1247e88ed227SDaniel Bristot de Oliveira {
1248e88ed227SDaniel Bristot de Oliveira if (timerlat_enabled() && !test_bit(OSN_WORKLOAD, &osnoise_options))
1249e88ed227SDaniel Bristot de Oliveira unregister_trace_sched_migrate_task(trace_sched_migrate_callback, NULL);
1250e88ed227SDaniel Bristot de Oliveira }
1251e88ed227SDaniel Bristot de Oliveira #else
register_migration_monitor(void)1252e88ed227SDaniel Bristot de Oliveira static int register_migration_monitor(void)
1253e88ed227SDaniel Bristot de Oliveira {
1254e88ed227SDaniel Bristot de Oliveira return 0;
1255e88ed227SDaniel Bristot de Oliveira }
unregister_migration_monitor(void)1256e88ed227SDaniel Bristot de Oliveira static void unregister_migration_monitor(void) {}
1257e88ed227SDaniel Bristot de Oliveira #endif
1258bce29ac9SDaniel Bristot de Oliveira /*
1259bce29ac9SDaniel Bristot de Oliveira * trace_sched_switch - sched:sched_switch trace event handler
1260bce29ac9SDaniel Bristot de Oliveira *
1261bce29ac9SDaniel Bristot de Oliveira * This function is hooked to the sched:sched_switch trace event, and it is
1262bce29ac9SDaniel Bristot de Oliveira * used to record the beginning and to report the end of a thread noise window.
1263bce29ac9SDaniel Bristot de Oliveira */
1264f7d9f637SDaniel Bristot de Oliveira static void
trace_sched_switch_callback(void * data,bool preempt,struct task_struct * p,struct task_struct * n,unsigned int prev_state)1265fa2c3254SValentin Schneider trace_sched_switch_callback(void *data, bool preempt,
1266fa2c3254SValentin Schneider struct task_struct *p,
12679c2136beSDelyan Kratunov struct task_struct *n,
12689c2136beSDelyan Kratunov unsigned int prev_state)
1269bce29ac9SDaniel Bristot de Oliveira {
1270bce29ac9SDaniel Bristot de Oliveira struct osnoise_variables *osn_var = this_cpu_osn_var();
127130838fcdSDaniel Bristot de Oliveira int workload = test_bit(OSN_WORKLOAD, &osnoise_options);
1272bce29ac9SDaniel Bristot de Oliveira
127330838fcdSDaniel Bristot de Oliveira if ((p->pid != osn_var->pid) || !workload)
1274bce29ac9SDaniel Bristot de Oliveira thread_exit(osn_var, p);
1275bce29ac9SDaniel Bristot de Oliveira
127630838fcdSDaniel Bristot de Oliveira if ((n->pid != osn_var->pid) || !workload)
1277bce29ac9SDaniel Bristot de Oliveira thread_entry(osn_var, n);
1278bce29ac9SDaniel Bristot de Oliveira }
1279bce29ac9SDaniel Bristot de Oliveira
1280bce29ac9SDaniel Bristot de Oliveira /*
1281e88ed227SDaniel Bristot de Oliveira * hook_thread_events - Hook the instrumentation for thread noise
1282bce29ac9SDaniel Bristot de Oliveira *
1283bce29ac9SDaniel Bristot de Oliveira * Hook the osnoise tracer callbacks to handle the noise from other
1284bce29ac9SDaniel Bristot de Oliveira * threads on the necessary kernel events.
1285bce29ac9SDaniel Bristot de Oliveira */
hook_thread_events(void)1286f7d9f637SDaniel Bristot de Oliveira static int hook_thread_events(void)
1287bce29ac9SDaniel Bristot de Oliveira {
1288bce29ac9SDaniel Bristot de Oliveira int ret;
1289bce29ac9SDaniel Bristot de Oliveira
1290bce29ac9SDaniel Bristot de Oliveira ret = register_trace_sched_switch(trace_sched_switch_callback, NULL);
1291bce29ac9SDaniel Bristot de Oliveira if (ret)
1292bce29ac9SDaniel Bristot de Oliveira return -EINVAL;
1293bce29ac9SDaniel Bristot de Oliveira
1294e88ed227SDaniel Bristot de Oliveira ret = register_migration_monitor();
1295e88ed227SDaniel Bristot de Oliveira if (ret)
1296e88ed227SDaniel Bristot de Oliveira goto out_unreg;
1297e88ed227SDaniel Bristot de Oliveira
1298bce29ac9SDaniel Bristot de Oliveira return 0;
1299e88ed227SDaniel Bristot de Oliveira
1300e88ed227SDaniel Bristot de Oliveira out_unreg:
1301e88ed227SDaniel Bristot de Oliveira unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1302e88ed227SDaniel Bristot de Oliveira return -EINVAL;
1303bce29ac9SDaniel Bristot de Oliveira }
1304bce29ac9SDaniel Bristot de Oliveira
1305bce29ac9SDaniel Bristot de Oliveira /*
1306e88ed227SDaniel Bristot de Oliveira * unhook_thread_events - unhook the instrumentation for thread noise
1307bce29ac9SDaniel Bristot de Oliveira *
1308bce29ac9SDaniel Bristot de Oliveira * Unook the osnoise tracer callbacks to handle the noise from other
1309bce29ac9SDaniel Bristot de Oliveira * threads on the necessary kernel events.
1310bce29ac9SDaniel Bristot de Oliveira */
unhook_thread_events(void)1311f7d9f637SDaniel Bristot de Oliveira static void unhook_thread_events(void)
1312bce29ac9SDaniel Bristot de Oliveira {
1313bce29ac9SDaniel Bristot de Oliveira unregister_trace_sched_switch(trace_sched_switch_callback, NULL);
1314e88ed227SDaniel Bristot de Oliveira unregister_migration_monitor();
1315bce29ac9SDaniel Bristot de Oliveira }
1316bce29ac9SDaniel Bristot de Oliveira
1317bce29ac9SDaniel Bristot de Oliveira /*
1318bce29ac9SDaniel Bristot de Oliveira * save_osn_sample_stats - Save the osnoise_sample statistics
1319bce29ac9SDaniel Bristot de Oliveira *
1320bce29ac9SDaniel Bristot de Oliveira * Save the osnoise_sample statistics before the sampling phase. These
1321bce29ac9SDaniel Bristot de Oliveira * values will be used later to compute the diff betwneen the statistics
1322bce29ac9SDaniel Bristot de Oliveira * before and after the osnoise sampling.
1323bce29ac9SDaniel Bristot de Oliveira */
1324f7d9f637SDaniel Bristot de Oliveira static void
save_osn_sample_stats(struct osnoise_variables * osn_var,struct osnoise_sample * s)1325f7d9f637SDaniel Bristot de Oliveira save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1326bce29ac9SDaniel Bristot de Oliveira {
1327bce29ac9SDaniel Bristot de Oliveira s->nmi_count = osn_var->nmi.count;
1328bce29ac9SDaniel Bristot de Oliveira s->irq_count = osn_var->irq.count;
1329bce29ac9SDaniel Bristot de Oliveira s->softirq_count = osn_var->softirq.count;
1330bce29ac9SDaniel Bristot de Oliveira s->thread_count = osn_var->thread.count;
1331bce29ac9SDaniel Bristot de Oliveira }
1332bce29ac9SDaniel Bristot de Oliveira
1333bce29ac9SDaniel Bristot de Oliveira /*
1334bce29ac9SDaniel Bristot de Oliveira * diff_osn_sample_stats - Compute the osnoise_sample statistics
1335bce29ac9SDaniel Bristot de Oliveira *
1336bce29ac9SDaniel Bristot de Oliveira * After a sample period, compute the difference on the osnoise_sample
1337bce29ac9SDaniel Bristot de Oliveira * statistics. The struct osnoise_sample *s contains the statistics saved via
1338bce29ac9SDaniel Bristot de Oliveira * save_osn_sample_stats() before the osnoise sampling.
1339bce29ac9SDaniel Bristot de Oliveira */
1340f7d9f637SDaniel Bristot de Oliveira static void
diff_osn_sample_stats(struct osnoise_variables * osn_var,struct osnoise_sample * s)1341f7d9f637SDaniel Bristot de Oliveira diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)
1342bce29ac9SDaniel Bristot de Oliveira {
1343bce29ac9SDaniel Bristot de Oliveira s->nmi_count = osn_var->nmi.count - s->nmi_count;
1344bce29ac9SDaniel Bristot de Oliveira s->irq_count = osn_var->irq.count - s->irq_count;
1345bce29ac9SDaniel Bristot de Oliveira s->softirq_count = osn_var->softirq.count - s->softirq_count;
1346bce29ac9SDaniel Bristot de Oliveira s->thread_count = osn_var->thread.count - s->thread_count;
1347bce29ac9SDaniel Bristot de Oliveira }
1348bce29ac9SDaniel Bristot de Oliveira
1349bce29ac9SDaniel Bristot de Oliveira /*
1350bce29ac9SDaniel Bristot de Oliveira * osnoise_stop_tracing - Stop tracing and the tracer.
1351bce29ac9SDaniel Bristot de Oliveira */
osnoise_stop_tracing(void)13520e05ba49SDaniel Bristot de Oliveira static __always_inline void osnoise_stop_tracing(void)
1353bce29ac9SDaniel Bristot de Oliveira {
1354dae18134SDaniel Bristot de Oliveira struct osnoise_instance *inst;
1355dae18134SDaniel Bristot de Oliveira struct trace_array *tr;
13560e05ba49SDaniel Bristot de Oliveira
1357dae18134SDaniel Bristot de Oliveira rcu_read_lock();
1358dae18134SDaniel Bristot de Oliveira list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1359dae18134SDaniel Bristot de Oliveira tr = inst->tr;
13600e05ba49SDaniel Bristot de Oliveira trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_,
13610e05ba49SDaniel Bristot de Oliveira "stop tracing hit on cpu %d\n", smp_processor_id());
13620e05ba49SDaniel Bristot de Oliveira
13631603dda4SDaniel Bristot de Oliveira if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options))
13641603dda4SDaniel Bristot de Oliveira panic("tracer hit stop condition on CPU %d\n", smp_processor_id());
13651603dda4SDaniel Bristot de Oliveira
1366bce29ac9SDaniel Bristot de Oliveira tracer_tracing_off(tr);
1367bce29ac9SDaniel Bristot de Oliveira }
1368dae18134SDaniel Bristot de Oliveira rcu_read_unlock();
1369dae18134SDaniel Bristot de Oliveira }
1370dae18134SDaniel Bristot de Oliveira
1371dae18134SDaniel Bristot de Oliveira /*
1372cb7ca871SDaniel Bristot de Oliveira * osnoise_has_tracing_on - Check if there is at least one instance on
1373cb7ca871SDaniel Bristot de Oliveira */
osnoise_has_tracing_on(void)1374cb7ca871SDaniel Bristot de Oliveira static __always_inline int osnoise_has_tracing_on(void)
1375cb7ca871SDaniel Bristot de Oliveira {
1376cb7ca871SDaniel Bristot de Oliveira struct osnoise_instance *inst;
1377cb7ca871SDaniel Bristot de Oliveira int trace_is_on = 0;
1378cb7ca871SDaniel Bristot de Oliveira
1379cb7ca871SDaniel Bristot de Oliveira rcu_read_lock();
1380cb7ca871SDaniel Bristot de Oliveira list_for_each_entry_rcu(inst, &osnoise_instances, list)
1381cb7ca871SDaniel Bristot de Oliveira trace_is_on += tracer_tracing_is_on(inst->tr);
1382cb7ca871SDaniel Bristot de Oliveira rcu_read_unlock();
1383cb7ca871SDaniel Bristot de Oliveira
1384cb7ca871SDaniel Bristot de Oliveira return trace_is_on;
1385cb7ca871SDaniel Bristot de Oliveira }
1386cb7ca871SDaniel Bristot de Oliveira
1387cb7ca871SDaniel Bristot de Oliveira /*
1388dae18134SDaniel Bristot de Oliveira * notify_new_max_latency - Notify a new max latency via fsnotify interface.
1389dae18134SDaniel Bristot de Oliveira */
notify_new_max_latency(u64 latency)1390dae18134SDaniel Bristot de Oliveira static void notify_new_max_latency(u64 latency)
1391dae18134SDaniel Bristot de Oliveira {
1392dae18134SDaniel Bristot de Oliveira struct osnoise_instance *inst;
1393dae18134SDaniel Bristot de Oliveira struct trace_array *tr;
1394dae18134SDaniel Bristot de Oliveira
1395dae18134SDaniel Bristot de Oliveira rcu_read_lock();
1396dae18134SDaniel Bristot de Oliveira list_for_each_entry_rcu(inst, &osnoise_instances, list) {
1397dae18134SDaniel Bristot de Oliveira tr = inst->tr;
1398d3cba7f0SDaniel Bristot de Oliveira if (tracer_tracing_is_on(tr) && tr->max_latency < latency) {
1399dae18134SDaniel Bristot de Oliveira tr->max_latency = latency;
1400dae18134SDaniel Bristot de Oliveira latency_fsnotify(tr);
1401dae18134SDaniel Bristot de Oliveira }
1402dae18134SDaniel Bristot de Oliveira }
1403dae18134SDaniel Bristot de Oliveira rcu_read_unlock();
1404dae18134SDaniel Bristot de Oliveira }
1405bce29ac9SDaniel Bristot de Oliveira
1406bce29ac9SDaniel Bristot de Oliveira /*
1407bce29ac9SDaniel Bristot de Oliveira * run_osnoise - Sample the time and look for osnoise
1408bce29ac9SDaniel Bristot de Oliveira *
1409bce29ac9SDaniel Bristot de Oliveira * Used to capture the time, looking for potential osnoise latency repeatedly.
1410bce29ac9SDaniel Bristot de Oliveira * Different from hwlat_detector, it is called with preemption and interrupts
1411bce29ac9SDaniel Bristot de Oliveira * enabled. This allows irqs, softirqs and threads to run, interfering on the
1412bce29ac9SDaniel Bristot de Oliveira * osnoise sampling thread, as they would do with a regular thread.
1413bce29ac9SDaniel Bristot de Oliveira */
run_osnoise(void)1414bce29ac9SDaniel Bristot de Oliveira static int run_osnoise(void)
1415bce29ac9SDaniel Bristot de Oliveira {
1416b5dce200SDaniel Bristot de Oliveira bool disable_irq = test_bit(OSN_IRQ_DISABLE, &osnoise_options);
1417bce29ac9SDaniel Bristot de Oliveira struct osnoise_variables *osn_var = this_cpu_osn_var();
1418bce29ac9SDaniel Bristot de Oliveira u64 start, sample, last_sample;
1419bce29ac9SDaniel Bristot de Oliveira u64 last_int_count, int_count;
142019c3eaa7SDaniel Bristot de Oliveira s64 noise = 0, max_noise = 0;
1421bce29ac9SDaniel Bristot de Oliveira s64 total, last_total = 0;
1422bce29ac9SDaniel Bristot de Oliveira struct osnoise_sample s;
1423b5dce200SDaniel Bristot de Oliveira bool disable_preemption;
1424bce29ac9SDaniel Bristot de Oliveira unsigned int threshold;
1425bce29ac9SDaniel Bristot de Oliveira u64 runtime, stop_in;
142619c3eaa7SDaniel Bristot de Oliveira u64 sum_noise = 0;
142719c3eaa7SDaniel Bristot de Oliveira int hw_count = 0;
1428bce29ac9SDaniel Bristot de Oliveira int ret = -1;
1429bce29ac9SDaniel Bristot de Oliveira
1430bce29ac9SDaniel Bristot de Oliveira /*
1431b5dce200SDaniel Bristot de Oliveira * Disabling preemption is only required if IRQs are enabled,
1432b5dce200SDaniel Bristot de Oliveira * and the options is set on.
1433b5dce200SDaniel Bristot de Oliveira */
1434b5dce200SDaniel Bristot de Oliveira disable_preemption = !disable_irq && test_bit(OSN_PREEMPT_DISABLE, &osnoise_options);
1435b5dce200SDaniel Bristot de Oliveira
1436b5dce200SDaniel Bristot de Oliveira /*
1437bce29ac9SDaniel Bristot de Oliveira * Considers the current thread as the workload.
1438bce29ac9SDaniel Bristot de Oliveira */
1439bce29ac9SDaniel Bristot de Oliveira osn_var->pid = current->pid;
1440bce29ac9SDaniel Bristot de Oliveira
1441bce29ac9SDaniel Bristot de Oliveira /*
1442bce29ac9SDaniel Bristot de Oliveira * Save the current stats for the diff
1443bce29ac9SDaniel Bristot de Oliveira */
1444bce29ac9SDaniel Bristot de Oliveira save_osn_sample_stats(osn_var, &s);
1445bce29ac9SDaniel Bristot de Oliveira
1446bce29ac9SDaniel Bristot de Oliveira /*
1447bce29ac9SDaniel Bristot de Oliveira * if threshold is 0, use the default value of 5 us.
1448bce29ac9SDaniel Bristot de Oliveira */
1449bce29ac9SDaniel Bristot de Oliveira threshold = tracing_thresh ? : 5000;
1450bce29ac9SDaniel Bristot de Oliveira
1451bce29ac9SDaniel Bristot de Oliveira /*
1452b5dce200SDaniel Bristot de Oliveira * Apply PREEMPT and IRQ disabled options.
1453b5dce200SDaniel Bristot de Oliveira */
1454b5dce200SDaniel Bristot de Oliveira if (disable_irq)
1455b5dce200SDaniel Bristot de Oliveira local_irq_disable();
1456b5dce200SDaniel Bristot de Oliveira
1457b5dce200SDaniel Bristot de Oliveira if (disable_preemption)
1458b5dce200SDaniel Bristot de Oliveira preempt_disable();
1459b5dce200SDaniel Bristot de Oliveira
1460b5dce200SDaniel Bristot de Oliveira /*
1461bce29ac9SDaniel Bristot de Oliveira * Make sure NMIs see sampling first
1462bce29ac9SDaniel Bristot de Oliveira */
1463bce29ac9SDaniel Bristot de Oliveira osn_var->sampling = true;
1464bce29ac9SDaniel Bristot de Oliveira barrier();
1465bce29ac9SDaniel Bristot de Oliveira
1466bce29ac9SDaniel Bristot de Oliveira /*
1467bce29ac9SDaniel Bristot de Oliveira * Transform the *_us config to nanoseconds to avoid the
1468bce29ac9SDaniel Bristot de Oliveira * division on the main loop.
1469bce29ac9SDaniel Bristot de Oliveira */
1470bce29ac9SDaniel Bristot de Oliveira runtime = osnoise_data.sample_runtime * NSEC_PER_USEC;
1471bce29ac9SDaniel Bristot de Oliveira stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC;
1472bce29ac9SDaniel Bristot de Oliveira
1473bce29ac9SDaniel Bristot de Oliveira /*
1474bce29ac9SDaniel Bristot de Oliveira * Start timestemp
1475bce29ac9SDaniel Bristot de Oliveira */
1476bce29ac9SDaniel Bristot de Oliveira start = time_get();
1477bce29ac9SDaniel Bristot de Oliveira
1478bce29ac9SDaniel Bristot de Oliveira /*
1479bce29ac9SDaniel Bristot de Oliveira * "previous" loop.
1480bce29ac9SDaniel Bristot de Oliveira */
1481bce29ac9SDaniel Bristot de Oliveira last_int_count = set_int_safe_time(osn_var, &last_sample);
1482bce29ac9SDaniel Bristot de Oliveira
1483bce29ac9SDaniel Bristot de Oliveira do {
1484bce29ac9SDaniel Bristot de Oliveira /*
1485bce29ac9SDaniel Bristot de Oliveira * Get sample!
1486bce29ac9SDaniel Bristot de Oliveira */
1487bce29ac9SDaniel Bristot de Oliveira int_count = set_int_safe_time(osn_var, &sample);
1488bce29ac9SDaniel Bristot de Oliveira
1489bce29ac9SDaniel Bristot de Oliveira noise = time_sub(sample, last_sample);
1490bce29ac9SDaniel Bristot de Oliveira
1491bce29ac9SDaniel Bristot de Oliveira /*
1492bce29ac9SDaniel Bristot de Oliveira * This shouldn't happen.
1493bce29ac9SDaniel Bristot de Oliveira */
1494bce29ac9SDaniel Bristot de Oliveira if (noise < 0) {
1495bce29ac9SDaniel Bristot de Oliveira osnoise_taint("negative noise!");
1496bce29ac9SDaniel Bristot de Oliveira goto out;
1497bce29ac9SDaniel Bristot de Oliveira }
1498bce29ac9SDaniel Bristot de Oliveira
1499bce29ac9SDaniel Bristot de Oliveira /*
1500bce29ac9SDaniel Bristot de Oliveira * Sample runtime.
1501bce29ac9SDaniel Bristot de Oliveira */
1502bce29ac9SDaniel Bristot de Oliveira total = time_sub(sample, start);
1503bce29ac9SDaniel Bristot de Oliveira
1504bce29ac9SDaniel Bristot de Oliveira /*
1505bce29ac9SDaniel Bristot de Oliveira * Check for possible overflows.
1506bce29ac9SDaniel Bristot de Oliveira */
1507bce29ac9SDaniel Bristot de Oliveira if (total < last_total) {
1508bce29ac9SDaniel Bristot de Oliveira osnoise_taint("total overflow!");
1509bce29ac9SDaniel Bristot de Oliveira break;
1510bce29ac9SDaniel Bristot de Oliveira }
1511bce29ac9SDaniel Bristot de Oliveira
1512bce29ac9SDaniel Bristot de Oliveira last_total = total;
1513bce29ac9SDaniel Bristot de Oliveira
1514bce29ac9SDaniel Bristot de Oliveira if (noise >= threshold) {
1515bce29ac9SDaniel Bristot de Oliveira int interference = int_count - last_int_count;
1516bce29ac9SDaniel Bristot de Oliveira
1517bce29ac9SDaniel Bristot de Oliveira if (noise > max_noise)
1518bce29ac9SDaniel Bristot de Oliveira max_noise = noise;
1519bce29ac9SDaniel Bristot de Oliveira
1520bce29ac9SDaniel Bristot de Oliveira if (!interference)
1521bce29ac9SDaniel Bristot de Oliveira hw_count++;
1522bce29ac9SDaniel Bristot de Oliveira
1523bce29ac9SDaniel Bristot de Oliveira sum_noise += noise;
1524bce29ac9SDaniel Bristot de Oliveira
1525bce29ac9SDaniel Bristot de Oliveira trace_sample_threshold(last_sample, noise, interference);
1526bce29ac9SDaniel Bristot de Oliveira
1527bce29ac9SDaniel Bristot de Oliveira if (osnoise_data.stop_tracing)
1528bce29ac9SDaniel Bristot de Oliveira if (noise > stop_in)
1529bce29ac9SDaniel Bristot de Oliveira osnoise_stop_tracing();
1530bce29ac9SDaniel Bristot de Oliveira }
1531bce29ac9SDaniel Bristot de Oliveira
1532bce29ac9SDaniel Bristot de Oliveira /*
1533caf4c86bSNicolas Saenz Julienne * In some cases, notably when running on a nohz_full CPU with
1534caf4c86bSNicolas Saenz Julienne * a stopped tick PREEMPT_RCU has no way to account for QSs.
1535caf4c86bSNicolas Saenz Julienne * This will eventually cause unwarranted noise as PREEMPT_RCU
1536caf4c86bSNicolas Saenz Julienne * will force preemption as the means of ending the current
1537caf4c86bSNicolas Saenz Julienne * grace period. We avoid this problem by calling
1538caf4c86bSNicolas Saenz Julienne * rcu_momentary_dyntick_idle(), which performs a zero duration
1539caf4c86bSNicolas Saenz Julienne * EQS allowing PREEMPT_RCU to end the current grace period.
1540caf4c86bSNicolas Saenz Julienne * This call shouldn't be wrapped inside an RCU critical
1541caf4c86bSNicolas Saenz Julienne * section.
1542caf4c86bSNicolas Saenz Julienne *
1543caf4c86bSNicolas Saenz Julienne * Note that in non PREEMPT_RCU kernels QSs are handled through
1544caf4c86bSNicolas Saenz Julienne * cond_resched()
1545caf4c86bSNicolas Saenz Julienne */
1546caf4c86bSNicolas Saenz Julienne if (IS_ENABLED(CONFIG_PREEMPT_RCU)) {
1547b5dce200SDaniel Bristot de Oliveira if (!disable_irq)
1548caf4c86bSNicolas Saenz Julienne local_irq_disable();
1549b5dce200SDaniel Bristot de Oliveira
1550caf4c86bSNicolas Saenz Julienne rcu_momentary_dyntick_idle();
1551b5dce200SDaniel Bristot de Oliveira
1552b5dce200SDaniel Bristot de Oliveira if (!disable_irq)
1553caf4c86bSNicolas Saenz Julienne local_irq_enable();
1554caf4c86bSNicolas Saenz Julienne }
1555caf4c86bSNicolas Saenz Julienne
1556caf4c86bSNicolas Saenz Julienne /*
1557bce29ac9SDaniel Bristot de Oliveira * For the non-preemptive kernel config: let threads runs, if
1558b5dce200SDaniel Bristot de Oliveira * they so wish, unless set not do to so.
1559bce29ac9SDaniel Bristot de Oliveira */
1560b5dce200SDaniel Bristot de Oliveira if (!disable_irq && !disable_preemption)
1561bce29ac9SDaniel Bristot de Oliveira cond_resched();
1562bce29ac9SDaniel Bristot de Oliveira
1563bce29ac9SDaniel Bristot de Oliveira last_sample = sample;
1564bce29ac9SDaniel Bristot de Oliveira last_int_count = int_count;
1565bce29ac9SDaniel Bristot de Oliveira
1566bce29ac9SDaniel Bristot de Oliveira } while (total < runtime && !kthread_should_stop());
1567bce29ac9SDaniel Bristot de Oliveira
1568bce29ac9SDaniel Bristot de Oliveira /*
1569bce29ac9SDaniel Bristot de Oliveira * Finish the above in the view for interrupts.
1570bce29ac9SDaniel Bristot de Oliveira */
1571bce29ac9SDaniel Bristot de Oliveira barrier();
1572bce29ac9SDaniel Bristot de Oliveira
1573bce29ac9SDaniel Bristot de Oliveira osn_var->sampling = false;
1574bce29ac9SDaniel Bristot de Oliveira
1575bce29ac9SDaniel Bristot de Oliveira /*
1576bce29ac9SDaniel Bristot de Oliveira * Make sure sampling data is no longer updated.
1577bce29ac9SDaniel Bristot de Oliveira */
1578bce29ac9SDaniel Bristot de Oliveira barrier();
1579bce29ac9SDaniel Bristot de Oliveira
1580bce29ac9SDaniel Bristot de Oliveira /*
1581b5dce200SDaniel Bristot de Oliveira * Return to the preemptive state.
1582b5dce200SDaniel Bristot de Oliveira */
1583b5dce200SDaniel Bristot de Oliveira if (disable_preemption)
1584b5dce200SDaniel Bristot de Oliveira preempt_enable();
1585b5dce200SDaniel Bristot de Oliveira
1586b5dce200SDaniel Bristot de Oliveira if (disable_irq)
1587b5dce200SDaniel Bristot de Oliveira local_irq_enable();
1588b5dce200SDaniel Bristot de Oliveira
1589b5dce200SDaniel Bristot de Oliveira /*
1590bce29ac9SDaniel Bristot de Oliveira * Save noise info.
1591bce29ac9SDaniel Bristot de Oliveira */
1592bce29ac9SDaniel Bristot de Oliveira s.noise = time_to_us(sum_noise);
1593bce29ac9SDaniel Bristot de Oliveira s.runtime = time_to_us(total);
1594bce29ac9SDaniel Bristot de Oliveira s.max_sample = time_to_us(max_noise);
1595bce29ac9SDaniel Bristot de Oliveira s.hw_count = hw_count;
1596bce29ac9SDaniel Bristot de Oliveira
1597bce29ac9SDaniel Bristot de Oliveira /* Save interference stats info */
1598bce29ac9SDaniel Bristot de Oliveira diff_osn_sample_stats(osn_var, &s);
1599bce29ac9SDaniel Bristot de Oliveira
1600bce29ac9SDaniel Bristot de Oliveira trace_osnoise_sample(&s);
1601bce29ac9SDaniel Bristot de Oliveira
1602dae18134SDaniel Bristot de Oliveira notify_new_max_latency(max_noise);
1603bce29ac9SDaniel Bristot de Oliveira
1604bce29ac9SDaniel Bristot de Oliveira if (osnoise_data.stop_tracing_total)
1605bce29ac9SDaniel Bristot de Oliveira if (s.noise > osnoise_data.stop_tracing_total)
1606bce29ac9SDaniel Bristot de Oliveira osnoise_stop_tracing();
1607bce29ac9SDaniel Bristot de Oliveira
1608bce29ac9SDaniel Bristot de Oliveira return 0;
1609bce29ac9SDaniel Bristot de Oliveira out:
1610bce29ac9SDaniel Bristot de Oliveira return ret;
1611bce29ac9SDaniel Bristot de Oliveira }
1612bce29ac9SDaniel Bristot de Oliveira
1613bce29ac9SDaniel Bristot de Oliveira static struct cpumask osnoise_cpumask;
1614bce29ac9SDaniel Bristot de Oliveira static struct cpumask save_cpumask;
1615bce29ac9SDaniel Bristot de Oliveira
1616bce29ac9SDaniel Bristot de Oliveira /*
1617dd990352SDaniel Bristot de Oliveira * osnoise_sleep - sleep until the next period
1618dd990352SDaniel Bristot de Oliveira */
osnoise_sleep(bool skip_period)1619cb7ca871SDaniel Bristot de Oliveira static void osnoise_sleep(bool skip_period)
1620dd990352SDaniel Bristot de Oliveira {
1621dd990352SDaniel Bristot de Oliveira u64 interval;
1622dd990352SDaniel Bristot de Oliveira ktime_t wake_time;
1623dd990352SDaniel Bristot de Oliveira
1624dd990352SDaniel Bristot de Oliveira mutex_lock(&interface_lock);
1625cb7ca871SDaniel Bristot de Oliveira if (skip_period)
1626cb7ca871SDaniel Bristot de Oliveira interval = osnoise_data.sample_period;
1627cb7ca871SDaniel Bristot de Oliveira else
1628dd990352SDaniel Bristot de Oliveira interval = osnoise_data.sample_period - osnoise_data.sample_runtime;
1629dd990352SDaniel Bristot de Oliveira mutex_unlock(&interface_lock);
1630dd990352SDaniel Bristot de Oliveira
1631dd990352SDaniel Bristot de Oliveira /*
1632dd990352SDaniel Bristot de Oliveira * differently from hwlat_detector, the osnoise tracer can run
1633dd990352SDaniel Bristot de Oliveira * without a pause because preemption is on.
1634dd990352SDaniel Bristot de Oliveira */
1635dd990352SDaniel Bristot de Oliveira if (!interval) {
1636dd990352SDaniel Bristot de Oliveira /* Let synchronize_rcu_tasks() make progress */
1637dd990352SDaniel Bristot de Oliveira cond_resched_tasks_rcu_qs();
1638dd990352SDaniel Bristot de Oliveira return;
1639dd990352SDaniel Bristot de Oliveira }
1640dd990352SDaniel Bristot de Oliveira
1641dd990352SDaniel Bristot de Oliveira wake_time = ktime_add_us(ktime_get(), interval);
1642dd990352SDaniel Bristot de Oliveira __set_current_state(TASK_INTERRUPTIBLE);
1643dd990352SDaniel Bristot de Oliveira
1644b18c58afSDavidlohr Bueso while (schedule_hrtimeout(&wake_time, HRTIMER_MODE_ABS)) {
1645dd990352SDaniel Bristot de Oliveira if (kthread_should_stop())
1646dd990352SDaniel Bristot de Oliveira break;
1647dd990352SDaniel Bristot de Oliveira }
1648dd990352SDaniel Bristot de Oliveira }
1649dd990352SDaniel Bristot de Oliveira
1650dd990352SDaniel Bristot de Oliveira /*
16514998e7fdSDaniel Bristot de Oliveira * osnoise_migration_pending - checks if the task needs to migrate
16524998e7fdSDaniel Bristot de Oliveira *
16534998e7fdSDaniel Bristot de Oliveira * osnoise/timerlat threads are per-cpu. If there is a pending request to
16544998e7fdSDaniel Bristot de Oliveira * migrate the thread away from the current CPU, something bad has happened.
16554998e7fdSDaniel Bristot de Oliveira * Play the good citizen and leave.
16564998e7fdSDaniel Bristot de Oliveira *
16574998e7fdSDaniel Bristot de Oliveira * Returns 0 if it is safe to continue, 1 otherwise.
16584998e7fdSDaniel Bristot de Oliveira */
osnoise_migration_pending(void)16594998e7fdSDaniel Bristot de Oliveira static inline int osnoise_migration_pending(void)
16604998e7fdSDaniel Bristot de Oliveira {
16614998e7fdSDaniel Bristot de Oliveira if (!current->migration_pending)
16624998e7fdSDaniel Bristot de Oliveira return 0;
16634998e7fdSDaniel Bristot de Oliveira
16644998e7fdSDaniel Bristot de Oliveira /*
16654998e7fdSDaniel Bristot de Oliveira * If migration is pending, there is a task waiting for the
16664998e7fdSDaniel Bristot de Oliveira * tracer to enable migration. The tracer does not allow migration,
16674998e7fdSDaniel Bristot de Oliveira * thus: taint and leave to unblock the blocked thread.
16684998e7fdSDaniel Bristot de Oliveira */
16694998e7fdSDaniel Bristot de Oliveira osnoise_taint("migration requested to osnoise threads, leaving.");
16704998e7fdSDaniel Bristot de Oliveira
16714998e7fdSDaniel Bristot de Oliveira /*
16724998e7fdSDaniel Bristot de Oliveira * Unset this thread from the threads managed by the interface.
16734998e7fdSDaniel Bristot de Oliveira * The tracers are responsible for cleaning their env before
16744998e7fdSDaniel Bristot de Oliveira * exiting.
16754998e7fdSDaniel Bristot de Oliveira */
16764998e7fdSDaniel Bristot de Oliveira mutex_lock(&interface_lock);
16774998e7fdSDaniel Bristot de Oliveira this_cpu_osn_var()->kthread = NULL;
16784998e7fdSDaniel Bristot de Oliveira mutex_unlock(&interface_lock);
16794998e7fdSDaniel Bristot de Oliveira
16804998e7fdSDaniel Bristot de Oliveira return 1;
16814998e7fdSDaniel Bristot de Oliveira }
16824998e7fdSDaniel Bristot de Oliveira
16834998e7fdSDaniel Bristot de Oliveira /*
1684bce29ac9SDaniel Bristot de Oliveira * osnoise_main - The osnoise detection kernel thread
1685bce29ac9SDaniel Bristot de Oliveira *
1686bce29ac9SDaniel Bristot de Oliveira * Calls run_osnoise() function to measure the osnoise for the configured runtime,
1687bce29ac9SDaniel Bristot de Oliveira * every period.
1688bce29ac9SDaniel Bristot de Oliveira */
osnoise_main(void * data)1689bce29ac9SDaniel Bristot de Oliveira static int osnoise_main(void *data)
1690bce29ac9SDaniel Bristot de Oliveira {
16914998e7fdSDaniel Bristot de Oliveira unsigned long flags;
16924998e7fdSDaniel Bristot de Oliveira
16934998e7fdSDaniel Bristot de Oliveira /*
16944998e7fdSDaniel Bristot de Oliveira * This thread was created pinned to the CPU using PF_NO_SETAFFINITY.
16954998e7fdSDaniel Bristot de Oliveira * The problem is that cgroup does not allow PF_NO_SETAFFINITY thread.
16964998e7fdSDaniel Bristot de Oliveira *
16974998e7fdSDaniel Bristot de Oliveira * To work around this limitation, disable migration and remove the
16984998e7fdSDaniel Bristot de Oliveira * flag.
16994998e7fdSDaniel Bristot de Oliveira */
17004998e7fdSDaniel Bristot de Oliveira migrate_disable();
17014998e7fdSDaniel Bristot de Oliveira raw_spin_lock_irqsave(¤t->pi_lock, flags);
17024998e7fdSDaniel Bristot de Oliveira current->flags &= ~(PF_NO_SETAFFINITY);
17034998e7fdSDaniel Bristot de Oliveira raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
1704bce29ac9SDaniel Bristot de Oliveira
1705bce29ac9SDaniel Bristot de Oliveira while (!kthread_should_stop()) {
17064998e7fdSDaniel Bristot de Oliveira if (osnoise_migration_pending())
17074998e7fdSDaniel Bristot de Oliveira break;
17084998e7fdSDaniel Bristot de Oliveira
1709cb7ca871SDaniel Bristot de Oliveira /* skip a period if tracing is off on all instances */
1710cb7ca871SDaniel Bristot de Oliveira if (!osnoise_has_tracing_on()) {
1711cb7ca871SDaniel Bristot de Oliveira osnoise_sleep(true);
1712cb7ca871SDaniel Bristot de Oliveira continue;
1713cb7ca871SDaniel Bristot de Oliveira }
1714cb7ca871SDaniel Bristot de Oliveira
1715bce29ac9SDaniel Bristot de Oliveira run_osnoise();
1716cb7ca871SDaniel Bristot de Oliveira osnoise_sleep(false);
1717bce29ac9SDaniel Bristot de Oliveira }
1718bce29ac9SDaniel Bristot de Oliveira
17194998e7fdSDaniel Bristot de Oliveira migrate_enable();
1720bce29ac9SDaniel Bristot de Oliveira return 0;
1721bce29ac9SDaniel Bristot de Oliveira }
1722bce29ac9SDaniel Bristot de Oliveira
1723a955d7eaSDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
1724a955d7eaSDaniel Bristot de Oliveira /*
1725a955d7eaSDaniel Bristot de Oliveira * timerlat_irq - hrtimer handler for timerlat.
1726a955d7eaSDaniel Bristot de Oliveira */
timerlat_irq(struct hrtimer * timer)1727a955d7eaSDaniel Bristot de Oliveira static enum hrtimer_restart timerlat_irq(struct hrtimer *timer)
1728a955d7eaSDaniel Bristot de Oliveira {
1729a955d7eaSDaniel Bristot de Oliveira struct osnoise_variables *osn_var = this_cpu_osn_var();
1730a955d7eaSDaniel Bristot de Oliveira struct timerlat_variables *tlat;
1731a955d7eaSDaniel Bristot de Oliveira struct timerlat_sample s;
1732a955d7eaSDaniel Bristot de Oliveira u64 now;
1733a955d7eaSDaniel Bristot de Oliveira u64 diff;
1734a955d7eaSDaniel Bristot de Oliveira
1735a955d7eaSDaniel Bristot de Oliveira /*
1736a955d7eaSDaniel Bristot de Oliveira * I am not sure if the timer was armed for this CPU. So, get
1737a955d7eaSDaniel Bristot de Oliveira * the timerlat struct from the timer itself, not from this
1738a955d7eaSDaniel Bristot de Oliveira * CPU.
1739a955d7eaSDaniel Bristot de Oliveira */
1740a955d7eaSDaniel Bristot de Oliveira tlat = container_of(timer, struct timerlat_variables, timer);
1741a955d7eaSDaniel Bristot de Oliveira
1742a955d7eaSDaniel Bristot de Oliveira now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1743a955d7eaSDaniel Bristot de Oliveira
1744a955d7eaSDaniel Bristot de Oliveira /*
1745a955d7eaSDaniel Bristot de Oliveira * Enable the osnoise: events for thread an softirq.
1746a955d7eaSDaniel Bristot de Oliveira */
1747a955d7eaSDaniel Bristot de Oliveira tlat->tracing_thread = true;
1748a955d7eaSDaniel Bristot de Oliveira
1749a955d7eaSDaniel Bristot de Oliveira osn_var->thread.arrival_time = time_get();
1750a955d7eaSDaniel Bristot de Oliveira
1751a955d7eaSDaniel Bristot de Oliveira /*
1752a955d7eaSDaniel Bristot de Oliveira * A hardirq is running: the timer IRQ. It is for sure preempting
1753a955d7eaSDaniel Bristot de Oliveira * a thread, and potentially preempting a softirq.
1754a955d7eaSDaniel Bristot de Oliveira *
1755a955d7eaSDaniel Bristot de Oliveira * At this point, it is not interesting to know the duration of the
1756a955d7eaSDaniel Bristot de Oliveira * preempted thread (and maybe softirq), but how much time they will
1757a955d7eaSDaniel Bristot de Oliveira * delay the beginning of the execution of the timer thread.
1758a955d7eaSDaniel Bristot de Oliveira *
1759a955d7eaSDaniel Bristot de Oliveira * To get the correct (net) delay added by the softirq, its delta_start
1760a955d7eaSDaniel Bristot de Oliveira * is set as the IRQ one. In this way, at the return of the IRQ, the delta
1761a955d7eaSDaniel Bristot de Oliveira * start of the sofitrq will be zeroed, accounting then only the time
1762a955d7eaSDaniel Bristot de Oliveira * after that.
1763a955d7eaSDaniel Bristot de Oliveira *
1764a955d7eaSDaniel Bristot de Oliveira * The thread follows the same principle. However, if a softirq is
1765a955d7eaSDaniel Bristot de Oliveira * running, the thread needs to receive the softirq delta_start. The
1766a955d7eaSDaniel Bristot de Oliveira * reason being is that the softirq will be the last to be unfolded,
1767a955d7eaSDaniel Bristot de Oliveira * resseting the thread delay to zero.
176801e181c7SDaniel Bristot de Oliveira *
176901e181c7SDaniel Bristot de Oliveira * The PREEMPT_RT is a special case, though. As softirqs run as threads
177001e181c7SDaniel Bristot de Oliveira * on RT, moving the thread is enough.
1771a955d7eaSDaniel Bristot de Oliveira */
177201e181c7SDaniel Bristot de Oliveira if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {
1773a955d7eaSDaniel Bristot de Oliveira copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1774a955d7eaSDaniel Bristot de Oliveira &osn_var->softirq.delta_start);
1775a955d7eaSDaniel Bristot de Oliveira
1776a955d7eaSDaniel Bristot de Oliveira copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,
1777a955d7eaSDaniel Bristot de Oliveira &osn_var->irq.delta_start);
1778a955d7eaSDaniel Bristot de Oliveira } else {
1779a955d7eaSDaniel Bristot de Oliveira copy_int_safe_time(osn_var, &osn_var->thread.delta_start,
1780a955d7eaSDaniel Bristot de Oliveira &osn_var->irq.delta_start);
1781a955d7eaSDaniel Bristot de Oliveira }
1782a955d7eaSDaniel Bristot de Oliveira
1783a955d7eaSDaniel Bristot de Oliveira /*
1784a955d7eaSDaniel Bristot de Oliveira * Compute the current time with the expected time.
1785a955d7eaSDaniel Bristot de Oliveira */
1786a955d7eaSDaniel Bristot de Oliveira diff = now - tlat->abs_period;
1787a955d7eaSDaniel Bristot de Oliveira
1788a955d7eaSDaniel Bristot de Oliveira tlat->count++;
1789a955d7eaSDaniel Bristot de Oliveira s.seqnum = tlat->count;
1790a955d7eaSDaniel Bristot de Oliveira s.timer_latency = diff;
1791a955d7eaSDaniel Bristot de Oliveira s.context = IRQ_CONTEXT;
1792a955d7eaSDaniel Bristot de Oliveira
1793a955d7eaSDaniel Bristot de Oliveira trace_timerlat_sample(&s);
1794a955d7eaSDaniel Bristot de Oliveira
1795aa748949SDaniel Bristot de Oliveira if (osnoise_data.stop_tracing) {
1796aa748949SDaniel Bristot de Oliveira if (time_to_us(diff) >= osnoise_data.stop_tracing) {
17974dd2aea2SDaniel Bristot de Oliveira
17984dd2aea2SDaniel Bristot de Oliveira /*
17994dd2aea2SDaniel Bristot de Oliveira * At this point, if stop_tracing is set and <= print_stack,
18004dd2aea2SDaniel Bristot de Oliveira * print_stack is set and would be printed in the thread handler.
18014dd2aea2SDaniel Bristot de Oliveira *
18024dd2aea2SDaniel Bristot de Oliveira * Thus, print the stack trace as it is helpful to define the
18034dd2aea2SDaniel Bristot de Oliveira * root cause of an IRQ latency.
18044dd2aea2SDaniel Bristot de Oliveira */
18054dd2aea2SDaniel Bristot de Oliveira if (osnoise_data.stop_tracing <= osnoise_data.print_stack) {
18064dd2aea2SDaniel Bristot de Oliveira timerlat_save_stack(0);
18074dd2aea2SDaniel Bristot de Oliveira timerlat_dump_stack(time_to_us(diff));
18084dd2aea2SDaniel Bristot de Oliveira }
18094dd2aea2SDaniel Bristot de Oliveira
1810a955d7eaSDaniel Bristot de Oliveira osnoise_stop_tracing();
1811a955d7eaSDaniel Bristot de Oliveira notify_new_max_latency(diff);
1812a955d7eaSDaniel Bristot de Oliveira
1813632478a0SDaniel Bristot de Oliveira wake_up_process(tlat->kthread);
1814632478a0SDaniel Bristot de Oliveira
18159c556e5aSDaniel Bristot de Oliveira return HRTIMER_NORESTART;
1816aa748949SDaniel Bristot de Oliveira }
1817aa748949SDaniel Bristot de Oliveira }
1818a955d7eaSDaniel Bristot de Oliveira
1819a955d7eaSDaniel Bristot de Oliveira wake_up_process(tlat->kthread);
1820a955d7eaSDaniel Bristot de Oliveira
1821a955d7eaSDaniel Bristot de Oliveira if (osnoise_data.print_stack)
1822a955d7eaSDaniel Bristot de Oliveira timerlat_save_stack(0);
1823a955d7eaSDaniel Bristot de Oliveira
1824a955d7eaSDaniel Bristot de Oliveira return HRTIMER_NORESTART;
1825a955d7eaSDaniel Bristot de Oliveira }
1826a955d7eaSDaniel Bristot de Oliveira
1827a955d7eaSDaniel Bristot de Oliveira /*
1828a955d7eaSDaniel Bristot de Oliveira * wait_next_period - Wait for the next period for timerlat
1829a955d7eaSDaniel Bristot de Oliveira */
wait_next_period(struct timerlat_variables * tlat)1830a955d7eaSDaniel Bristot de Oliveira static int wait_next_period(struct timerlat_variables *tlat)
1831a955d7eaSDaniel Bristot de Oliveira {
1832a955d7eaSDaniel Bristot de Oliveira ktime_t next_abs_period, now;
1833a955d7eaSDaniel Bristot de Oliveira u64 rel_period = osnoise_data.timerlat_period * 1000;
1834a955d7eaSDaniel Bristot de Oliveira
1835a955d7eaSDaniel Bristot de Oliveira now = hrtimer_cb_get_time(&tlat->timer);
1836a955d7eaSDaniel Bristot de Oliveira next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1837a955d7eaSDaniel Bristot de Oliveira
1838a955d7eaSDaniel Bristot de Oliveira /*
1839a955d7eaSDaniel Bristot de Oliveira * Save the next abs_period.
1840a955d7eaSDaniel Bristot de Oliveira */
1841a955d7eaSDaniel Bristot de Oliveira tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1842a955d7eaSDaniel Bristot de Oliveira
1843a955d7eaSDaniel Bristot de Oliveira /*
1844a955d7eaSDaniel Bristot de Oliveira * If the new abs_period is in the past, skip the activation.
1845a955d7eaSDaniel Bristot de Oliveira */
1846a955d7eaSDaniel Bristot de Oliveira while (ktime_compare(now, next_abs_period) > 0) {
1847a955d7eaSDaniel Bristot de Oliveira next_abs_period = ns_to_ktime(tlat->abs_period + rel_period);
1848a955d7eaSDaniel Bristot de Oliveira tlat->abs_period = (u64) ktime_to_ns(next_abs_period);
1849a955d7eaSDaniel Bristot de Oliveira }
1850a955d7eaSDaniel Bristot de Oliveira
1851a955d7eaSDaniel Bristot de Oliveira set_current_state(TASK_INTERRUPTIBLE);
1852a955d7eaSDaniel Bristot de Oliveira
1853a955d7eaSDaniel Bristot de Oliveira hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD);
1854a955d7eaSDaniel Bristot de Oliveira schedule();
1855a955d7eaSDaniel Bristot de Oliveira return 1;
1856a955d7eaSDaniel Bristot de Oliveira }
1857a955d7eaSDaniel Bristot de Oliveira
1858a955d7eaSDaniel Bristot de Oliveira /*
1859a955d7eaSDaniel Bristot de Oliveira * timerlat_main- Timerlat main
1860a955d7eaSDaniel Bristot de Oliveira */
timerlat_main(void * data)1861a955d7eaSDaniel Bristot de Oliveira static int timerlat_main(void *data)
1862a955d7eaSDaniel Bristot de Oliveira {
1863a955d7eaSDaniel Bristot de Oliveira struct osnoise_variables *osn_var = this_cpu_osn_var();
1864a955d7eaSDaniel Bristot de Oliveira struct timerlat_variables *tlat = this_cpu_tmr_var();
1865a955d7eaSDaniel Bristot de Oliveira struct timerlat_sample s;
1866a955d7eaSDaniel Bristot de Oliveira struct sched_param sp;
18674998e7fdSDaniel Bristot de Oliveira unsigned long flags;
1868a955d7eaSDaniel Bristot de Oliveira u64 now, diff;
1869a955d7eaSDaniel Bristot de Oliveira
1870a955d7eaSDaniel Bristot de Oliveira /*
1871a955d7eaSDaniel Bristot de Oliveira * Make the thread RT, that is how cyclictest is usually used.
1872a955d7eaSDaniel Bristot de Oliveira */
1873a955d7eaSDaniel Bristot de Oliveira sp.sched_priority = DEFAULT_TIMERLAT_PRIO;
1874a955d7eaSDaniel Bristot de Oliveira sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
1875a955d7eaSDaniel Bristot de Oliveira
18764998e7fdSDaniel Bristot de Oliveira /*
18774998e7fdSDaniel Bristot de Oliveira * This thread was created pinned to the CPU using PF_NO_SETAFFINITY.
18784998e7fdSDaniel Bristot de Oliveira * The problem is that cgroup does not allow PF_NO_SETAFFINITY thread.
18794998e7fdSDaniel Bristot de Oliveira *
18804998e7fdSDaniel Bristot de Oliveira * To work around this limitation, disable migration and remove the
18814998e7fdSDaniel Bristot de Oliveira * flag.
18824998e7fdSDaniel Bristot de Oliveira */
18834998e7fdSDaniel Bristot de Oliveira migrate_disable();
18844998e7fdSDaniel Bristot de Oliveira raw_spin_lock_irqsave(¤t->pi_lock, flags);
18854998e7fdSDaniel Bristot de Oliveira current->flags &= ~(PF_NO_SETAFFINITY);
18864998e7fdSDaniel Bristot de Oliveira raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
18874998e7fdSDaniel Bristot de Oliveira
1888a955d7eaSDaniel Bristot de Oliveira tlat->count = 0;
1889a955d7eaSDaniel Bristot de Oliveira tlat->tracing_thread = false;
1890a955d7eaSDaniel Bristot de Oliveira
1891a955d7eaSDaniel Bristot de Oliveira hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1892a955d7eaSDaniel Bristot de Oliveira tlat->timer.function = timerlat_irq;
1893a955d7eaSDaniel Bristot de Oliveira tlat->kthread = current;
1894a955d7eaSDaniel Bristot de Oliveira osn_var->pid = current->pid;
1895a955d7eaSDaniel Bristot de Oliveira /*
1896a955d7eaSDaniel Bristot de Oliveira * Anotate the arrival time.
1897a955d7eaSDaniel Bristot de Oliveira */
1898a955d7eaSDaniel Bristot de Oliveira tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
1899a955d7eaSDaniel Bristot de Oliveira
1900a955d7eaSDaniel Bristot de Oliveira wait_next_period(tlat);
1901a955d7eaSDaniel Bristot de Oliveira
1902a955d7eaSDaniel Bristot de Oliveira osn_var->sampling = 1;
1903a955d7eaSDaniel Bristot de Oliveira
1904a955d7eaSDaniel Bristot de Oliveira while (!kthread_should_stop()) {
19054998e7fdSDaniel Bristot de Oliveira
1906a955d7eaSDaniel Bristot de Oliveira now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
1907a955d7eaSDaniel Bristot de Oliveira diff = now - tlat->abs_period;
1908a955d7eaSDaniel Bristot de Oliveira
1909a955d7eaSDaniel Bristot de Oliveira s.seqnum = tlat->count;
1910a955d7eaSDaniel Bristot de Oliveira s.timer_latency = diff;
1911a955d7eaSDaniel Bristot de Oliveira s.context = THREAD_CONTEXT;
1912a955d7eaSDaniel Bristot de Oliveira
1913a955d7eaSDaniel Bristot de Oliveira trace_timerlat_sample(&s);
1914a955d7eaSDaniel Bristot de Oliveira
1915b9f451a9SDaniel Bristot de Oliveira notify_new_max_latency(diff);
1916b9f451a9SDaniel Bristot de Oliveira
1917b14f4568SDaniel Bristot de Oliveira timerlat_dump_stack(time_to_us(diff));
1918a955d7eaSDaniel Bristot de Oliveira
1919a955d7eaSDaniel Bristot de Oliveira tlat->tracing_thread = false;
1920a955d7eaSDaniel Bristot de Oliveira if (osnoise_data.stop_tracing_total)
1921a955d7eaSDaniel Bristot de Oliveira if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
1922a955d7eaSDaniel Bristot de Oliveira osnoise_stop_tracing();
1923a955d7eaSDaniel Bristot de Oliveira
19244998e7fdSDaniel Bristot de Oliveira if (osnoise_migration_pending())
19254998e7fdSDaniel Bristot de Oliveira break;
19264998e7fdSDaniel Bristot de Oliveira
1927a955d7eaSDaniel Bristot de Oliveira wait_next_period(tlat);
1928a955d7eaSDaniel Bristot de Oliveira }
1929a955d7eaSDaniel Bristot de Oliveira
1930a955d7eaSDaniel Bristot de Oliveira hrtimer_cancel(&tlat->timer);
19314998e7fdSDaniel Bristot de Oliveira migrate_enable();
1932a955d7eaSDaniel Bristot de Oliveira return 0;
1933a955d7eaSDaniel Bristot de Oliveira }
1934ccb67544SDaniel Bristot de Oliveira #else /* CONFIG_TIMERLAT_TRACER */
timerlat_main(void * data)1935ccb67544SDaniel Bristot de Oliveira static int timerlat_main(void *data)
1936ccb67544SDaniel Bristot de Oliveira {
1937ccb67544SDaniel Bristot de Oliveira return 0;
1938ccb67544SDaniel Bristot de Oliveira }
1939a955d7eaSDaniel Bristot de Oliveira #endif /* CONFIG_TIMERLAT_TRACER */
1940a955d7eaSDaniel Bristot de Oliveira
1941bce29ac9SDaniel Bristot de Oliveira /*
1942c8895e27SDaniel Bristot de Oliveira * stop_kthread - stop a workload thread
1943c8895e27SDaniel Bristot de Oliveira */
stop_kthread(unsigned int cpu)1944c8895e27SDaniel Bristot de Oliveira static void stop_kthread(unsigned int cpu)
1945c8895e27SDaniel Bristot de Oliveira {
1946c8895e27SDaniel Bristot de Oliveira struct task_struct *kthread;
1947c8895e27SDaniel Bristot de Oliveira
1948c8895e27SDaniel Bristot de Oliveira kthread = per_cpu(per_cpu_osnoise_var, cpu).kthread;
194930838fcdSDaniel Bristot de Oliveira if (kthread) {
1950e88ed227SDaniel Bristot de Oliveira if (test_bit(OSN_WORKLOAD, &osnoise_options)) {
1951c8895e27SDaniel Bristot de Oliveira kthread_stop(kthread);
1952e88ed227SDaniel Bristot de Oliveira } else {
1953e88ed227SDaniel Bristot de Oliveira /*
1954e88ed227SDaniel Bristot de Oliveira * This is a user thread waiting on the timerlat_fd. We need
1955e88ed227SDaniel Bristot de Oliveira * to close all users, and the best way to guarantee this is
1956e88ed227SDaniel Bristot de Oliveira * by killing the thread. NOTE: this is a purpose specific file.
1957e88ed227SDaniel Bristot de Oliveira */
1958e88ed227SDaniel Bristot de Oliveira kill_pid(kthread->thread_pid, SIGKILL, 1);
1959e88ed227SDaniel Bristot de Oliveira put_task_struct(kthread);
1960e88ed227SDaniel Bristot de Oliveira }
1961c8895e27SDaniel Bristot de Oliveira per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
196230838fcdSDaniel Bristot de Oliveira } else {
1963e88ed227SDaniel Bristot de Oliveira /* if no workload, just return */
196430838fcdSDaniel Bristot de Oliveira if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
1965e88ed227SDaniel Bristot de Oliveira /*
1966e88ed227SDaniel Bristot de Oliveira * This is set in the osnoise tracer case.
1967e88ed227SDaniel Bristot de Oliveira */
196830838fcdSDaniel Bristot de Oliveira per_cpu(per_cpu_osnoise_var, cpu).sampling = false;
196930838fcdSDaniel Bristot de Oliveira barrier();
197030838fcdSDaniel Bristot de Oliveira return;
197130838fcdSDaniel Bristot de Oliveira }
197230838fcdSDaniel Bristot de Oliveira }
1973c8895e27SDaniel Bristot de Oliveira }
1974c8895e27SDaniel Bristot de Oliveira
1975c8895e27SDaniel Bristot de Oliveira /*
1976c8895e27SDaniel Bristot de Oliveira * stop_per_cpu_kthread - Stop per-cpu threads
1977bce29ac9SDaniel Bristot de Oliveira *
1978bce29ac9SDaniel Bristot de Oliveira * Stop the osnoise sampling htread. Use this on unload and at system
1979bce29ac9SDaniel Bristot de Oliveira * shutdown.
1980bce29ac9SDaniel Bristot de Oliveira */
stop_per_cpu_kthreads(void)1981bce29ac9SDaniel Bristot de Oliveira static void stop_per_cpu_kthreads(void)
1982bce29ac9SDaniel Bristot de Oliveira {
1983bce29ac9SDaniel Bristot de Oliveira int cpu;
1984bce29ac9SDaniel Bristot de Oliveira
198599c37d1aSSebastian Andrzej Siewior cpus_read_lock();
1986bce29ac9SDaniel Bristot de Oliveira
1987bce29ac9SDaniel Bristot de Oliveira for_each_online_cpu(cpu)
1988c8895e27SDaniel Bristot de Oliveira stop_kthread(cpu);
1989bce29ac9SDaniel Bristot de Oliveira
199099c37d1aSSebastian Andrzej Siewior cpus_read_unlock();
1991c8895e27SDaniel Bristot de Oliveira }
1992c8895e27SDaniel Bristot de Oliveira
1993c8895e27SDaniel Bristot de Oliveira /*
1994c8895e27SDaniel Bristot de Oliveira * start_kthread - Start a workload tread
1995c8895e27SDaniel Bristot de Oliveira */
start_kthread(unsigned int cpu)1996c8895e27SDaniel Bristot de Oliveira static int start_kthread(unsigned int cpu)
1997c8895e27SDaniel Bristot de Oliveira {
1998c8895e27SDaniel Bristot de Oliveira struct task_struct *kthread;
1999c8895e27SDaniel Bristot de Oliveira void *main = osnoise_main;
2000c8895e27SDaniel Bristot de Oliveira char comm[24];
2001c8895e27SDaniel Bristot de Oliveira
2002ccb67544SDaniel Bristot de Oliveira if (timerlat_enabled()) {
2003a955d7eaSDaniel Bristot de Oliveira snprintf(comm, 24, "timerlat/%d", cpu);
2004a955d7eaSDaniel Bristot de Oliveira main = timerlat_main;
2005a955d7eaSDaniel Bristot de Oliveira } else {
200630838fcdSDaniel Bristot de Oliveira /* if no workload, just return */
200730838fcdSDaniel Bristot de Oliveira if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
200830838fcdSDaniel Bristot de Oliveira per_cpu(per_cpu_osnoise_var, cpu).sampling = true;
200930838fcdSDaniel Bristot de Oliveira barrier();
201030838fcdSDaniel Bristot de Oliveira return 0;
201130838fcdSDaniel Bristot de Oliveira }
2012bce29ac9SDaniel Bristot de Oliveira snprintf(comm, 24, "osnoise/%d", cpu);
2013a955d7eaSDaniel Bristot de Oliveira }
2014ccb67544SDaniel Bristot de Oliveira
201511e4e352SCai Huoqing kthread = kthread_run_on_cpu(main, NULL, cpu, comm);
2016bce29ac9SDaniel Bristot de Oliveira
2017bce29ac9SDaniel Bristot de Oliveira if (IS_ERR(kthread)) {
2018bce29ac9SDaniel Bristot de Oliveira pr_err(BANNER "could not start sampling thread\n");
2019bce29ac9SDaniel Bristot de Oliveira stop_per_cpu_kthreads();
2020bce29ac9SDaniel Bristot de Oliveira return -ENOMEM;
2021bce29ac9SDaniel Bristot de Oliveira }
2022bce29ac9SDaniel Bristot de Oliveira
2023bce29ac9SDaniel Bristot de Oliveira per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread;
2024bce29ac9SDaniel Bristot de Oliveira
2025bce29ac9SDaniel Bristot de Oliveira return 0;
2026bce29ac9SDaniel Bristot de Oliveira }
2027bce29ac9SDaniel Bristot de Oliveira
2028bce29ac9SDaniel Bristot de Oliveira /*
2029c8895e27SDaniel Bristot de Oliveira * start_per_cpu_kthread - Kick off per-cpu osnoise sampling kthreads
2030c8895e27SDaniel Bristot de Oliveira *
2031c8895e27SDaniel Bristot de Oliveira * This starts the kernel thread that will look for osnoise on many
2032c8895e27SDaniel Bristot de Oliveira * cpus.
2033c8895e27SDaniel Bristot de Oliveira */
start_per_cpu_kthreads(void)203415ca4bdbSDaniel Bristot de Oliveira static int start_per_cpu_kthreads(void)
2035c8895e27SDaniel Bristot de Oliveira {
2036c8895e27SDaniel Bristot de Oliveira struct cpumask *current_mask = &save_cpumask;
20374b6b08f2SQiang.Zhang int retval = 0;
2038c8895e27SDaniel Bristot de Oliveira int cpu;
2039c8895e27SDaniel Bristot de Oliveira
2040e88ed227SDaniel Bristot de Oliveira if (!test_bit(OSN_WORKLOAD, &osnoise_options)) {
2041e88ed227SDaniel Bristot de Oliveira if (timerlat_enabled())
2042e88ed227SDaniel Bristot de Oliveira return 0;
2043e88ed227SDaniel Bristot de Oliveira }
2044e88ed227SDaniel Bristot de Oliveira
204599c37d1aSSebastian Andrzej Siewior cpus_read_lock();
2046c8895e27SDaniel Bristot de Oliveira /*
204766df27f1SDaniel Bristot de Oliveira * Run only on online CPUs in which osnoise is allowed to run.
2048c8895e27SDaniel Bristot de Oliveira */
204966df27f1SDaniel Bristot de Oliveira cpumask_and(current_mask, cpu_online_mask, &osnoise_cpumask);
2050c8895e27SDaniel Bristot de Oliveira
2051c8895e27SDaniel Bristot de Oliveira for_each_possible_cpu(cpu)
2052c8895e27SDaniel Bristot de Oliveira per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;
2053c8895e27SDaniel Bristot de Oliveira
2054c8895e27SDaniel Bristot de Oliveira for_each_cpu(cpu, current_mask) {
2055c8895e27SDaniel Bristot de Oliveira retval = start_kthread(cpu);
2056c8895e27SDaniel Bristot de Oliveira if (retval) {
205799ee9317SNico Pache cpus_read_unlock();
2058c8895e27SDaniel Bristot de Oliveira stop_per_cpu_kthreads();
205999ee9317SNico Pache return retval;
2060c8895e27SDaniel Bristot de Oliveira }
2061c8895e27SDaniel Bristot de Oliveira }
2062c8895e27SDaniel Bristot de Oliveira
206399c37d1aSSebastian Andrzej Siewior cpus_read_unlock();
2064c8895e27SDaniel Bristot de Oliveira
20654b6b08f2SQiang.Zhang return retval;
2066c8895e27SDaniel Bristot de Oliveira }
2067c8895e27SDaniel Bristot de Oliveira
2068c8895e27SDaniel Bristot de Oliveira #ifdef CONFIG_HOTPLUG_CPU
osnoise_hotplug_workfn(struct work_struct * dummy)2069c8895e27SDaniel Bristot de Oliveira static void osnoise_hotplug_workfn(struct work_struct *dummy)
2070c8895e27SDaniel Bristot de Oliveira {
2071c8895e27SDaniel Bristot de Oliveira unsigned int cpu = smp_processor_id();
2072c8895e27SDaniel Bristot de Oliveira
2073c8895e27SDaniel Bristot de Oliveira mutex_lock(&trace_types_lock);
2074c8895e27SDaniel Bristot de Oliveira
2075dae18134SDaniel Bristot de Oliveira if (!osnoise_has_registered_instances())
2076c8895e27SDaniel Bristot de Oliveira goto out_unlock_trace;
2077c8895e27SDaniel Bristot de Oliveira
2078c8895e27SDaniel Bristot de Oliveira mutex_lock(&interface_lock);
207999c37d1aSSebastian Andrzej Siewior cpus_read_lock();
2080c8895e27SDaniel Bristot de Oliveira
2081c8895e27SDaniel Bristot de Oliveira if (!cpumask_test_cpu(cpu, &osnoise_cpumask))
2082c8895e27SDaniel Bristot de Oliveira goto out_unlock;
2083c8895e27SDaniel Bristot de Oliveira
2084c8895e27SDaniel Bristot de Oliveira start_kthread(cpu);
2085c8895e27SDaniel Bristot de Oliveira
2086c8895e27SDaniel Bristot de Oliveira out_unlock:
208799c37d1aSSebastian Andrzej Siewior cpus_read_unlock();
2088c8895e27SDaniel Bristot de Oliveira mutex_unlock(&interface_lock);
2089c8895e27SDaniel Bristot de Oliveira out_unlock_trace:
2090c8895e27SDaniel Bristot de Oliveira mutex_unlock(&trace_types_lock);
2091c8895e27SDaniel Bristot de Oliveira }
2092c8895e27SDaniel Bristot de Oliveira
2093c8895e27SDaniel Bristot de Oliveira static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn);
2094c8895e27SDaniel Bristot de Oliveira
2095c8895e27SDaniel Bristot de Oliveira /*
2096c8895e27SDaniel Bristot de Oliveira * osnoise_cpu_init - CPU hotplug online callback function
2097c8895e27SDaniel Bristot de Oliveira */
osnoise_cpu_init(unsigned int cpu)2098c8895e27SDaniel Bristot de Oliveira static int osnoise_cpu_init(unsigned int cpu)
2099c8895e27SDaniel Bristot de Oliveira {
2100c8895e27SDaniel Bristot de Oliveira schedule_work_on(cpu, &osnoise_hotplug_work);
2101c8895e27SDaniel Bristot de Oliveira return 0;
2102c8895e27SDaniel Bristot de Oliveira }
2103c8895e27SDaniel Bristot de Oliveira
2104c8895e27SDaniel Bristot de Oliveira /*
2105c8895e27SDaniel Bristot de Oliveira * osnoise_cpu_die - CPU hotplug offline callback function
2106c8895e27SDaniel Bristot de Oliveira */
osnoise_cpu_die(unsigned int cpu)2107c8895e27SDaniel Bristot de Oliveira static int osnoise_cpu_die(unsigned int cpu)
2108c8895e27SDaniel Bristot de Oliveira {
2109c8895e27SDaniel Bristot de Oliveira stop_kthread(cpu);
2110c8895e27SDaniel Bristot de Oliveira return 0;
2111c8895e27SDaniel Bristot de Oliveira }
2112c8895e27SDaniel Bristot de Oliveira
osnoise_init_hotplug_support(void)2113c8895e27SDaniel Bristot de Oliveira static void osnoise_init_hotplug_support(void)
2114c8895e27SDaniel Bristot de Oliveira {
2115c8895e27SDaniel Bristot de Oliveira int ret;
2116c8895e27SDaniel Bristot de Oliveira
2117c8895e27SDaniel Bristot de Oliveira ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online",
2118c8895e27SDaniel Bristot de Oliveira osnoise_cpu_init, osnoise_cpu_die);
2119c8895e27SDaniel Bristot de Oliveira if (ret < 0)
2120c8895e27SDaniel Bristot de Oliveira pr_warn(BANNER "Error to init cpu hotplug support\n");
2121c8895e27SDaniel Bristot de Oliveira
2122c8895e27SDaniel Bristot de Oliveira return;
2123c8895e27SDaniel Bristot de Oliveira }
2124c8895e27SDaniel Bristot de Oliveira #else /* CONFIG_HOTPLUG_CPU */
osnoise_init_hotplug_support(void)2125c8895e27SDaniel Bristot de Oliveira static void osnoise_init_hotplug_support(void)
2126c8895e27SDaniel Bristot de Oliveira {
2127498627b4SDaniel Bristot de Oliveira return;
2128c8895e27SDaniel Bristot de Oliveira }
2129c8895e27SDaniel Bristot de Oliveira #endif /* CONFIG_HOTPLUG_CPU */
2130c8895e27SDaniel Bristot de Oliveira
2131c8895e27SDaniel Bristot de Oliveira /*
2132b179d48bSDaniel Bristot de Oliveira * seq file functions for the osnoise/options file.
2133b179d48bSDaniel Bristot de Oliveira */
s_options_start(struct seq_file * s,loff_t * pos)2134b179d48bSDaniel Bristot de Oliveira static void *s_options_start(struct seq_file *s, loff_t *pos)
2135b179d48bSDaniel Bristot de Oliveira {
2136b179d48bSDaniel Bristot de Oliveira int option = *pos;
2137b179d48bSDaniel Bristot de Oliveira
2138b179d48bSDaniel Bristot de Oliveira mutex_lock(&interface_lock);
2139b179d48bSDaniel Bristot de Oliveira
2140b179d48bSDaniel Bristot de Oliveira if (option >= OSN_MAX)
2141b179d48bSDaniel Bristot de Oliveira return NULL;
2142b179d48bSDaniel Bristot de Oliveira
2143b179d48bSDaniel Bristot de Oliveira return pos;
2144b179d48bSDaniel Bristot de Oliveira }
2145b179d48bSDaniel Bristot de Oliveira
s_options_next(struct seq_file * s,void * v,loff_t * pos)2146b179d48bSDaniel Bristot de Oliveira static void *s_options_next(struct seq_file *s, void *v, loff_t *pos)
2147b179d48bSDaniel Bristot de Oliveira {
2148b179d48bSDaniel Bristot de Oliveira int option = ++(*pos);
2149b179d48bSDaniel Bristot de Oliveira
2150b179d48bSDaniel Bristot de Oliveira if (option >= OSN_MAX)
2151b179d48bSDaniel Bristot de Oliveira return NULL;
2152b179d48bSDaniel Bristot de Oliveira
2153b179d48bSDaniel Bristot de Oliveira return pos;
2154b179d48bSDaniel Bristot de Oliveira }
2155b179d48bSDaniel Bristot de Oliveira
s_options_show(struct seq_file * s,void * v)2156b179d48bSDaniel Bristot de Oliveira static int s_options_show(struct seq_file *s, void *v)
2157b179d48bSDaniel Bristot de Oliveira {
2158b179d48bSDaniel Bristot de Oliveira loff_t *pos = v;
2159b179d48bSDaniel Bristot de Oliveira int option = *pos;
2160b179d48bSDaniel Bristot de Oliveira
2161b179d48bSDaniel Bristot de Oliveira if (option == OSN_DEFAULTS) {
2162b179d48bSDaniel Bristot de Oliveira if (osnoise_options == OSN_DEFAULT_OPTIONS)
2163b179d48bSDaniel Bristot de Oliveira seq_printf(s, "%s", osnoise_options_str[option]);
2164b179d48bSDaniel Bristot de Oliveira else
2165b179d48bSDaniel Bristot de Oliveira seq_printf(s, "NO_%s", osnoise_options_str[option]);
2166b179d48bSDaniel Bristot de Oliveira goto out;
2167b179d48bSDaniel Bristot de Oliveira }
2168b179d48bSDaniel Bristot de Oliveira
2169b179d48bSDaniel Bristot de Oliveira if (test_bit(option, &osnoise_options))
2170b179d48bSDaniel Bristot de Oliveira seq_printf(s, "%s", osnoise_options_str[option]);
2171b179d48bSDaniel Bristot de Oliveira else
2172b179d48bSDaniel Bristot de Oliveira seq_printf(s, "NO_%s", osnoise_options_str[option]);
2173b179d48bSDaniel Bristot de Oliveira
2174b179d48bSDaniel Bristot de Oliveira out:
2175b179d48bSDaniel Bristot de Oliveira if (option != OSN_MAX)
2176b179d48bSDaniel Bristot de Oliveira seq_puts(s, " ");
2177b179d48bSDaniel Bristot de Oliveira
2178b179d48bSDaniel Bristot de Oliveira return 0;
2179b179d48bSDaniel Bristot de Oliveira }
2180b179d48bSDaniel Bristot de Oliveira
s_options_stop(struct seq_file * s,void * v)2181b179d48bSDaniel Bristot de Oliveira static void s_options_stop(struct seq_file *s, void *v)
2182b179d48bSDaniel Bristot de Oliveira {
2183b179d48bSDaniel Bristot de Oliveira seq_puts(s, "\n");
2184b179d48bSDaniel Bristot de Oliveira mutex_unlock(&interface_lock);
2185b179d48bSDaniel Bristot de Oliveira }
2186b179d48bSDaniel Bristot de Oliveira
2187b179d48bSDaniel Bristot de Oliveira static const struct seq_operations osnoise_options_seq_ops = {
2188b179d48bSDaniel Bristot de Oliveira .start = s_options_start,
2189b179d48bSDaniel Bristot de Oliveira .next = s_options_next,
2190b179d48bSDaniel Bristot de Oliveira .show = s_options_show,
2191b179d48bSDaniel Bristot de Oliveira .stop = s_options_stop
2192b179d48bSDaniel Bristot de Oliveira };
2193b179d48bSDaniel Bristot de Oliveira
osnoise_options_open(struct inode * inode,struct file * file)2194b179d48bSDaniel Bristot de Oliveira static int osnoise_options_open(struct inode *inode, struct file *file)
2195b179d48bSDaniel Bristot de Oliveira {
2196b179d48bSDaniel Bristot de Oliveira return seq_open(file, &osnoise_options_seq_ops);
2197b179d48bSDaniel Bristot de Oliveira };
2198b179d48bSDaniel Bristot de Oliveira
2199b179d48bSDaniel Bristot de Oliveira /**
2200b179d48bSDaniel Bristot de Oliveira * osnoise_options_write - Write function for "options" entry
2201b179d48bSDaniel Bristot de Oliveira * @filp: The active open file structure
2202b179d48bSDaniel Bristot de Oliveira * @ubuf: The user buffer that contains the value to write
2203b179d48bSDaniel Bristot de Oliveira * @cnt: The maximum number of bytes to write to "file"
2204b179d48bSDaniel Bristot de Oliveira * @ppos: The current position in @file
2205b179d48bSDaniel Bristot de Oliveira *
2206b179d48bSDaniel Bristot de Oliveira * Writing the option name sets the option, writing the "NO_"
2207b179d48bSDaniel Bristot de Oliveira * prefix in front of the option name disables it.
2208b179d48bSDaniel Bristot de Oliveira *
2209b179d48bSDaniel Bristot de Oliveira * Writing "DEFAULTS" resets the option values to the default ones.
2210b179d48bSDaniel Bristot de Oliveira */
osnoise_options_write(struct file * filp,const char __user * ubuf,size_t cnt,loff_t * ppos)2211b179d48bSDaniel Bristot de Oliveira static ssize_t osnoise_options_write(struct file *filp, const char __user *ubuf,
2212b179d48bSDaniel Bristot de Oliveira size_t cnt, loff_t *ppos)
2213b179d48bSDaniel Bristot de Oliveira {
2214b179d48bSDaniel Bristot de Oliveira int running, option, enable, retval;
2215b179d48bSDaniel Bristot de Oliveira char buf[256], *option_str;
2216b179d48bSDaniel Bristot de Oliveira
2217b179d48bSDaniel Bristot de Oliveira if (cnt >= 256)
2218b179d48bSDaniel Bristot de Oliveira return -EINVAL;
2219b179d48bSDaniel Bristot de Oliveira
2220b179d48bSDaniel Bristot de Oliveira if (copy_from_user(buf, ubuf, cnt))
2221b179d48bSDaniel Bristot de Oliveira return -EFAULT;
2222b179d48bSDaniel Bristot de Oliveira
2223b179d48bSDaniel Bristot de Oliveira buf[cnt] = 0;
2224b179d48bSDaniel Bristot de Oliveira
2225b179d48bSDaniel Bristot de Oliveira if (strncmp(buf, "NO_", 3)) {
2226b179d48bSDaniel Bristot de Oliveira option_str = strstrip(buf);
2227b179d48bSDaniel Bristot de Oliveira enable = true;
2228b179d48bSDaniel Bristot de Oliveira } else {
2229b179d48bSDaniel Bristot de Oliveira option_str = strstrip(&buf[3]);
2230b179d48bSDaniel Bristot de Oliveira enable = false;
2231b179d48bSDaniel Bristot de Oliveira }
2232b179d48bSDaniel Bristot de Oliveira
2233b179d48bSDaniel Bristot de Oliveira option = match_string(osnoise_options_str, OSN_MAX, option_str);
2234b179d48bSDaniel Bristot de Oliveira if (option < 0)
2235b179d48bSDaniel Bristot de Oliveira return -EINVAL;
2236b179d48bSDaniel Bristot de Oliveira
2237b179d48bSDaniel Bristot de Oliveira /*
2238b179d48bSDaniel Bristot de Oliveira * trace_types_lock is taken to avoid concurrency on start/stop.
2239b179d48bSDaniel Bristot de Oliveira */
2240b179d48bSDaniel Bristot de Oliveira mutex_lock(&trace_types_lock);
2241b179d48bSDaniel Bristot de Oliveira running = osnoise_has_registered_instances();
2242b179d48bSDaniel Bristot de Oliveira if (running)
2243b179d48bSDaniel Bristot de Oliveira stop_per_cpu_kthreads();
2244b179d48bSDaniel Bristot de Oliveira
2245b179d48bSDaniel Bristot de Oliveira mutex_lock(&interface_lock);
2246b179d48bSDaniel Bristot de Oliveira /*
2247b179d48bSDaniel Bristot de Oliveira * avoid CPU hotplug operations that might read options.
2248b179d48bSDaniel Bristot de Oliveira */
2249b179d48bSDaniel Bristot de Oliveira cpus_read_lock();
2250b179d48bSDaniel Bristot de Oliveira
2251b179d48bSDaniel Bristot de Oliveira retval = cnt;
2252b179d48bSDaniel Bristot de Oliveira
2253b179d48bSDaniel Bristot de Oliveira if (enable) {
2254b179d48bSDaniel Bristot de Oliveira if (option == OSN_DEFAULTS)
2255b179d48bSDaniel Bristot de Oliveira osnoise_options = OSN_DEFAULT_OPTIONS;
2256b179d48bSDaniel Bristot de Oliveira else
2257b179d48bSDaniel Bristot de Oliveira set_bit(option, &osnoise_options);
2258b179d48bSDaniel Bristot de Oliveira } else {
2259b179d48bSDaniel Bristot de Oliveira if (option == OSN_DEFAULTS)
2260b179d48bSDaniel Bristot de Oliveira retval = -EINVAL;
2261b179d48bSDaniel Bristot de Oliveira else
2262b179d48bSDaniel Bristot de Oliveira clear_bit(option, &osnoise_options);
2263b179d48bSDaniel Bristot de Oliveira }
2264b179d48bSDaniel Bristot de Oliveira
2265b179d48bSDaniel Bristot de Oliveira cpus_read_unlock();
2266b179d48bSDaniel Bristot de Oliveira mutex_unlock(&interface_lock);
2267b179d48bSDaniel Bristot de Oliveira
2268b179d48bSDaniel Bristot de Oliveira if (running)
2269b179d48bSDaniel Bristot de Oliveira start_per_cpu_kthreads();
2270b179d48bSDaniel Bristot de Oliveira mutex_unlock(&trace_types_lock);
2271b179d48bSDaniel Bristot de Oliveira
2272b179d48bSDaniel Bristot de Oliveira return retval;
2273b179d48bSDaniel Bristot de Oliveira }
2274b179d48bSDaniel Bristot de Oliveira
2275b179d48bSDaniel Bristot de Oliveira /*
2276bce29ac9SDaniel Bristot de Oliveira * osnoise_cpus_read - Read function for reading the "cpus" file
2277bce29ac9SDaniel Bristot de Oliveira * @filp: The active open file structure
2278bce29ac9SDaniel Bristot de Oliveira * @ubuf: The userspace provided buffer to read value into
2279bce29ac9SDaniel Bristot de Oliveira * @cnt: The maximum number of bytes to read
2280bce29ac9SDaniel Bristot de Oliveira * @ppos: The current "file" position
2281bce29ac9SDaniel Bristot de Oliveira *
2282bce29ac9SDaniel Bristot de Oliveira * Prints the "cpus" output into the user-provided buffer.
2283bce29ac9SDaniel Bristot de Oliveira */
2284bce29ac9SDaniel Bristot de Oliveira static ssize_t
osnoise_cpus_read(struct file * filp,char __user * ubuf,size_t count,loff_t * ppos)2285bce29ac9SDaniel Bristot de Oliveira osnoise_cpus_read(struct file *filp, char __user *ubuf, size_t count,
2286bce29ac9SDaniel Bristot de Oliveira loff_t *ppos)
2287bce29ac9SDaniel Bristot de Oliveira {
2288bce29ac9SDaniel Bristot de Oliveira char *mask_str;
2289bce29ac9SDaniel Bristot de Oliveira int len;
2290bce29ac9SDaniel Bristot de Oliveira
2291bce29ac9SDaniel Bristot de Oliveira mutex_lock(&interface_lock);
2292bce29ac9SDaniel Bristot de Oliveira
2293bce29ac9SDaniel Bristot de Oliveira len = snprintf(NULL, 0, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask)) + 1;
2294bce29ac9SDaniel Bristot de Oliveira mask_str = kmalloc(len, GFP_KERNEL);
2295bce29ac9SDaniel Bristot de Oliveira if (!mask_str) {
2296bce29ac9SDaniel Bristot de Oliveira count = -ENOMEM;
2297bce29ac9SDaniel Bristot de Oliveira goto out_unlock;
2298bce29ac9SDaniel Bristot de Oliveira }
2299bce29ac9SDaniel Bristot de Oliveira
2300bce29ac9SDaniel Bristot de Oliveira len = snprintf(mask_str, len, "%*pbl\n", cpumask_pr_args(&osnoise_cpumask));
2301bce29ac9SDaniel Bristot de Oliveira if (len >= count) {
2302bce29ac9SDaniel Bristot de Oliveira count = -EINVAL;
2303bce29ac9SDaniel Bristot de Oliveira goto out_free;
2304bce29ac9SDaniel Bristot de Oliveira }
2305bce29ac9SDaniel Bristot de Oliveira
2306bce29ac9SDaniel Bristot de Oliveira count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
2307bce29ac9SDaniel Bristot de Oliveira
2308bce29ac9SDaniel Bristot de Oliveira out_free:
2309bce29ac9SDaniel Bristot de Oliveira kfree(mask_str);
2310bce29ac9SDaniel Bristot de Oliveira out_unlock:
2311bce29ac9SDaniel Bristot de Oliveira mutex_unlock(&interface_lock);
2312bce29ac9SDaniel Bristot de Oliveira
2313bce29ac9SDaniel Bristot de Oliveira return count;
2314bce29ac9SDaniel Bristot de Oliveira }
2315bce29ac9SDaniel Bristot de Oliveira
2316bce29ac9SDaniel Bristot de Oliveira /*
2317bce29ac9SDaniel Bristot de Oliveira * osnoise_cpus_write - Write function for "cpus" entry
2318bce29ac9SDaniel Bristot de Oliveira * @filp: The active open file structure
2319bce29ac9SDaniel Bristot de Oliveira * @ubuf: The user buffer that contains the value to write
2320bce29ac9SDaniel Bristot de Oliveira * @cnt: The maximum number of bytes to write to "file"
2321bce29ac9SDaniel Bristot de Oliveira * @ppos: The current position in @file
2322bce29ac9SDaniel Bristot de Oliveira *
2323bce29ac9SDaniel Bristot de Oliveira * This function provides a write implementation for the "cpus"
2324bce29ac9SDaniel Bristot de Oliveira * interface to the osnoise trace. By default, it lists all CPUs,
2325bce29ac9SDaniel Bristot de Oliveira * in this way, allowing osnoise threads to run on any online CPU
2326bce29ac9SDaniel Bristot de Oliveira * of the system. It serves to restrict the execution of osnoise to the
232766df27f1SDaniel Bristot de Oliveira * set of CPUs writing via this interface. Why not use "tracing_cpumask"?
232866df27f1SDaniel Bristot de Oliveira * Because the user might be interested in tracing what is running on
232966df27f1SDaniel Bristot de Oliveira * other CPUs. For instance, one might run osnoise in one HT CPU
233066df27f1SDaniel Bristot de Oliveira * while observing what is running on the sibling HT CPU.
2331bce29ac9SDaniel Bristot de Oliveira */
2332bce29ac9SDaniel Bristot de Oliveira static ssize_t
osnoise_cpus_write(struct file * filp,const char __user * ubuf,size_t count,loff_t * ppos)2333bce29ac9SDaniel Bristot de Oliveira osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count,
2334bce29ac9SDaniel Bristot de Oliveira loff_t *ppos)
2335bce29ac9SDaniel Bristot de Oliveira {
2336bce29ac9SDaniel Bristot de Oliveira cpumask_var_t osnoise_cpumask_new;
2337bce29ac9SDaniel Bristot de Oliveira int running, err;
2338bce29ac9SDaniel Bristot de Oliveira char buf[256];
2339bce29ac9SDaniel Bristot de Oliveira
2340bce29ac9SDaniel Bristot de Oliveira if (count >= 256)
2341bce29ac9SDaniel Bristot de Oliveira return -EINVAL;
2342bce29ac9SDaniel Bristot de Oliveira
2343bce29ac9SDaniel Bristot de Oliveira if (copy_from_user(buf, ubuf, count))
2344bce29ac9SDaniel Bristot de Oliveira return -EFAULT;
2345bce29ac9SDaniel Bristot de Oliveira
2346bce29ac9SDaniel Bristot de Oliveira if (!zalloc_cpumask_var(&osnoise_cpumask_new, GFP_KERNEL))
2347bce29ac9SDaniel Bristot de Oliveira return -ENOMEM;
2348bce29ac9SDaniel Bristot de Oliveira
2349bce29ac9SDaniel Bristot de Oliveira err = cpulist_parse(buf, osnoise_cpumask_new);
2350bce29ac9SDaniel Bristot de Oliveira if (err)
2351bce29ac9SDaniel Bristot de Oliveira goto err_free;
2352bce29ac9SDaniel Bristot de Oliveira
2353bce29ac9SDaniel Bristot de Oliveira /*
2354dae18134SDaniel Bristot de Oliveira * trace_types_lock is taken to avoid concurrency on start/stop.
2355bce29ac9SDaniel Bristot de Oliveira */
2356bce29ac9SDaniel Bristot de Oliveira mutex_lock(&trace_types_lock);
2357dae18134SDaniel Bristot de Oliveira running = osnoise_has_registered_instances();
2358bce29ac9SDaniel Bristot de Oliveira if (running)
23592bd1bdf0SDaniel Bristot de Oliveira stop_per_cpu_kthreads();
2360bce29ac9SDaniel Bristot de Oliveira
2361bce29ac9SDaniel Bristot de Oliveira mutex_lock(&interface_lock);
2362c8895e27SDaniel Bristot de Oliveira /*
2363c8895e27SDaniel Bristot de Oliveira * osnoise_cpumask is read by CPU hotplug operations.
2364c8895e27SDaniel Bristot de Oliveira */
236599c37d1aSSebastian Andrzej Siewior cpus_read_lock();
2366c8895e27SDaniel Bristot de Oliveira
2367bce29ac9SDaniel Bristot de Oliveira cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new);
2368c8895e27SDaniel Bristot de Oliveira
236999c37d1aSSebastian Andrzej Siewior cpus_read_unlock();
2370bce29ac9SDaniel Bristot de Oliveira mutex_unlock(&interface_lock);
2371bce29ac9SDaniel Bristot de Oliveira
2372bce29ac9SDaniel Bristot de Oliveira if (running)
23732bd1bdf0SDaniel Bristot de Oliveira start_per_cpu_kthreads();
2374bce29ac9SDaniel Bristot de Oliveira mutex_unlock(&trace_types_lock);
2375bce29ac9SDaniel Bristot de Oliveira
2376bce29ac9SDaniel Bristot de Oliveira free_cpumask_var(osnoise_cpumask_new);
2377bce29ac9SDaniel Bristot de Oliveira return count;
2378bce29ac9SDaniel Bristot de Oliveira
2379bce29ac9SDaniel Bristot de Oliveira err_free:
2380bce29ac9SDaniel Bristot de Oliveira free_cpumask_var(osnoise_cpumask_new);
2381bce29ac9SDaniel Bristot de Oliveira
2382bce29ac9SDaniel Bristot de Oliveira return err;
2383bce29ac9SDaniel Bristot de Oliveira }
2384bce29ac9SDaniel Bristot de Oliveira
2385e88ed227SDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
timerlat_fd_open(struct inode * inode,struct file * file)2386e88ed227SDaniel Bristot de Oliveira static int timerlat_fd_open(struct inode *inode, struct file *file)
2387e88ed227SDaniel Bristot de Oliveira {
2388e88ed227SDaniel Bristot de Oliveira struct osnoise_variables *osn_var;
2389e88ed227SDaniel Bristot de Oliveira struct timerlat_variables *tlat;
2390e88ed227SDaniel Bristot de Oliveira long cpu = (long) inode->i_cdev;
2391e88ed227SDaniel Bristot de Oliveira
2392e88ed227SDaniel Bristot de Oliveira mutex_lock(&interface_lock);
2393e88ed227SDaniel Bristot de Oliveira
2394e88ed227SDaniel Bristot de Oliveira /*
2395e88ed227SDaniel Bristot de Oliveira * This file is accessible only if timerlat is enabled, and
2396e88ed227SDaniel Bristot de Oliveira * NO_OSNOISE_WORKLOAD is set.
2397e88ed227SDaniel Bristot de Oliveira */
2398e88ed227SDaniel Bristot de Oliveira if (!timerlat_enabled() || test_bit(OSN_WORKLOAD, &osnoise_options)) {
2399e88ed227SDaniel Bristot de Oliveira mutex_unlock(&interface_lock);
2400e88ed227SDaniel Bristot de Oliveira return -EINVAL;
2401e88ed227SDaniel Bristot de Oliveira }
2402e88ed227SDaniel Bristot de Oliveira
2403e88ed227SDaniel Bristot de Oliveira migrate_disable();
2404e88ed227SDaniel Bristot de Oliveira
2405e88ed227SDaniel Bristot de Oliveira osn_var = this_cpu_osn_var();
2406e88ed227SDaniel Bristot de Oliveira
2407e88ed227SDaniel Bristot de Oliveira /*
2408e88ed227SDaniel Bristot de Oliveira * The osn_var->pid holds the single access to this file.
2409e88ed227SDaniel Bristot de Oliveira */
2410e88ed227SDaniel Bristot de Oliveira if (osn_var->pid) {
2411e88ed227SDaniel Bristot de Oliveira mutex_unlock(&interface_lock);
2412e88ed227SDaniel Bristot de Oliveira migrate_enable();
2413e88ed227SDaniel Bristot de Oliveira return -EBUSY;
2414e88ed227SDaniel Bristot de Oliveira }
2415e88ed227SDaniel Bristot de Oliveira
2416e88ed227SDaniel Bristot de Oliveira /*
2417e88ed227SDaniel Bristot de Oliveira * timerlat tracer is a per-cpu tracer. Check if the user-space too
2418e88ed227SDaniel Bristot de Oliveira * is pinned to a single CPU. The tracer laters monitor if the task
2419e88ed227SDaniel Bristot de Oliveira * migrates and then disables tracer if it does. However, it is
2420e88ed227SDaniel Bristot de Oliveira * worth doing this basic acceptance test to avoid obviusly wrong
2421e88ed227SDaniel Bristot de Oliveira * setup.
2422e88ed227SDaniel Bristot de Oliveira */
2423e88ed227SDaniel Bristot de Oliveira if (current->nr_cpus_allowed > 1 || cpu != smp_processor_id()) {
2424e88ed227SDaniel Bristot de Oliveira mutex_unlock(&interface_lock);
2425e88ed227SDaniel Bristot de Oliveira migrate_enable();
2426e88ed227SDaniel Bristot de Oliveira return -EPERM;
2427e88ed227SDaniel Bristot de Oliveira }
2428e88ed227SDaniel Bristot de Oliveira
2429e88ed227SDaniel Bristot de Oliveira /*
2430e88ed227SDaniel Bristot de Oliveira * From now on, it is good to go.
2431e88ed227SDaniel Bristot de Oliveira */
2432e88ed227SDaniel Bristot de Oliveira file->private_data = inode->i_cdev;
2433e88ed227SDaniel Bristot de Oliveira
2434e88ed227SDaniel Bristot de Oliveira get_task_struct(current);
2435e88ed227SDaniel Bristot de Oliveira
2436e88ed227SDaniel Bristot de Oliveira osn_var->kthread = current;
2437e88ed227SDaniel Bristot de Oliveira osn_var->pid = current->pid;
2438e88ed227SDaniel Bristot de Oliveira
2439e88ed227SDaniel Bristot de Oliveira /*
2440e88ed227SDaniel Bristot de Oliveira * Setup is done.
2441e88ed227SDaniel Bristot de Oliveira */
2442e88ed227SDaniel Bristot de Oliveira mutex_unlock(&interface_lock);
2443e88ed227SDaniel Bristot de Oliveira
2444e88ed227SDaniel Bristot de Oliveira tlat = this_cpu_tmr_var();
2445e88ed227SDaniel Bristot de Oliveira tlat->count = 0;
2446e88ed227SDaniel Bristot de Oliveira
2447*5f703935SDaniel Bristot de Oliveira hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
2448*5f703935SDaniel Bristot de Oliveira tlat->timer.function = timerlat_irq;
2449*5f703935SDaniel Bristot de Oliveira
2450e88ed227SDaniel Bristot de Oliveira migrate_enable();
2451e88ed227SDaniel Bristot de Oliveira return 0;
2452e88ed227SDaniel Bristot de Oliveira };
2453e88ed227SDaniel Bristot de Oliveira
2454e88ed227SDaniel Bristot de Oliveira /*
2455e88ed227SDaniel Bristot de Oliveira * timerlat_fd_read - Read function for "timerlat_fd" file
2456e88ed227SDaniel Bristot de Oliveira * @file: The active open file structure
2457e88ed227SDaniel Bristot de Oliveira * @ubuf: The userspace provided buffer to read value into
2458e88ed227SDaniel Bristot de Oliveira * @cnt: The maximum number of bytes to read
2459e88ed227SDaniel Bristot de Oliveira * @ppos: The current "file" position
2460e88ed227SDaniel Bristot de Oliveira *
2461e88ed227SDaniel Bristot de Oliveira * Prints 1 on timerlat, the number of interferences on osnoise, -1 on error.
2462e88ed227SDaniel Bristot de Oliveira */
2463e88ed227SDaniel Bristot de Oliveira static ssize_t
timerlat_fd_read(struct file * file,char __user * ubuf,size_t count,loff_t * ppos)2464e88ed227SDaniel Bristot de Oliveira timerlat_fd_read(struct file *file, char __user *ubuf, size_t count,
2465e88ed227SDaniel Bristot de Oliveira loff_t *ppos)
2466e88ed227SDaniel Bristot de Oliveira {
2467e88ed227SDaniel Bristot de Oliveira long cpu = (long) file->private_data;
2468e88ed227SDaniel Bristot de Oliveira struct osnoise_variables *osn_var;
2469e88ed227SDaniel Bristot de Oliveira struct timerlat_variables *tlat;
2470e88ed227SDaniel Bristot de Oliveira struct timerlat_sample s;
2471e88ed227SDaniel Bristot de Oliveira s64 diff;
2472e88ed227SDaniel Bristot de Oliveira u64 now;
2473e88ed227SDaniel Bristot de Oliveira
2474e88ed227SDaniel Bristot de Oliveira migrate_disable();
2475e88ed227SDaniel Bristot de Oliveira
2476e88ed227SDaniel Bristot de Oliveira tlat = this_cpu_tmr_var();
2477e88ed227SDaniel Bristot de Oliveira
2478e88ed227SDaniel Bristot de Oliveira /*
2479e88ed227SDaniel Bristot de Oliveira * While in user-space, the thread is migratable. There is nothing
2480e88ed227SDaniel Bristot de Oliveira * we can do about it.
2481e88ed227SDaniel Bristot de Oliveira * So, if the thread is running on another CPU, stop the machinery.
2482e88ed227SDaniel Bristot de Oliveira */
2483e88ed227SDaniel Bristot de Oliveira if (cpu == smp_processor_id()) {
2484e88ed227SDaniel Bristot de Oliveira if (tlat->uthread_migrate) {
2485e88ed227SDaniel Bristot de Oliveira migrate_enable();
2486e88ed227SDaniel Bristot de Oliveira return -EINVAL;
2487e88ed227SDaniel Bristot de Oliveira }
2488e88ed227SDaniel Bristot de Oliveira } else {
2489e88ed227SDaniel Bristot de Oliveira per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1;
2490e88ed227SDaniel Bristot de Oliveira osnoise_taint("timerlat user thread migrate\n");
2491e88ed227SDaniel Bristot de Oliveira osnoise_stop_tracing();
2492e88ed227SDaniel Bristot de Oliveira migrate_enable();
2493e88ed227SDaniel Bristot de Oliveira return -EINVAL;
2494e88ed227SDaniel Bristot de Oliveira }
2495e88ed227SDaniel Bristot de Oliveira
2496e88ed227SDaniel Bristot de Oliveira osn_var = this_cpu_osn_var();
2497e88ed227SDaniel Bristot de Oliveira
2498e88ed227SDaniel Bristot de Oliveira /*
2499e88ed227SDaniel Bristot de Oliveira * The timerlat in user-space runs in a different order:
2500e88ed227SDaniel Bristot de Oliveira * the read() starts from the execution of the previous occurrence,
2501e88ed227SDaniel Bristot de Oliveira * sleeping for the next occurrence.
2502e88ed227SDaniel Bristot de Oliveira *
2503e88ed227SDaniel Bristot de Oliveira * So, skip if we are entering on read() before the first wakeup
2504e88ed227SDaniel Bristot de Oliveira * from timerlat IRQ:
2505e88ed227SDaniel Bristot de Oliveira */
2506e88ed227SDaniel Bristot de Oliveira if (likely(osn_var->sampling)) {
2507e88ed227SDaniel Bristot de Oliveira now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
2508e88ed227SDaniel Bristot de Oliveira diff = now - tlat->abs_period;
2509e88ed227SDaniel Bristot de Oliveira
2510e88ed227SDaniel Bristot de Oliveira /*
2511e88ed227SDaniel Bristot de Oliveira * it was not a timer firing, but some other signal?
2512e88ed227SDaniel Bristot de Oliveira */
2513e88ed227SDaniel Bristot de Oliveira if (diff < 0)
2514e88ed227SDaniel Bristot de Oliveira goto out;
2515e88ed227SDaniel Bristot de Oliveira
2516e88ed227SDaniel Bristot de Oliveira s.seqnum = tlat->count;
2517e88ed227SDaniel Bristot de Oliveira s.timer_latency = diff;
2518e88ed227SDaniel Bristot de Oliveira s.context = THREAD_URET;
2519e88ed227SDaniel Bristot de Oliveira
2520e88ed227SDaniel Bristot de Oliveira trace_timerlat_sample(&s);
2521e88ed227SDaniel Bristot de Oliveira
2522e88ed227SDaniel Bristot de Oliveira notify_new_max_latency(diff);
2523e88ed227SDaniel Bristot de Oliveira
2524e88ed227SDaniel Bristot de Oliveira tlat->tracing_thread = false;
2525e88ed227SDaniel Bristot de Oliveira if (osnoise_data.stop_tracing_total)
2526e88ed227SDaniel Bristot de Oliveira if (time_to_us(diff) >= osnoise_data.stop_tracing_total)
2527e88ed227SDaniel Bristot de Oliveira osnoise_stop_tracing();
2528e88ed227SDaniel Bristot de Oliveira } else {
2529e88ed227SDaniel Bristot de Oliveira tlat->tracing_thread = false;
2530e88ed227SDaniel Bristot de Oliveira tlat->kthread = current;
2531e88ed227SDaniel Bristot de Oliveira
2532e88ed227SDaniel Bristot de Oliveira /* Annotate now to drift new period */
2533e88ed227SDaniel Bristot de Oliveira tlat->abs_period = hrtimer_cb_get_time(&tlat->timer);
2534e88ed227SDaniel Bristot de Oliveira
2535e88ed227SDaniel Bristot de Oliveira osn_var->sampling = 1;
2536e88ed227SDaniel Bristot de Oliveira }
2537e88ed227SDaniel Bristot de Oliveira
2538e88ed227SDaniel Bristot de Oliveira /* wait for the next period */
2539e88ed227SDaniel Bristot de Oliveira wait_next_period(tlat);
2540e88ed227SDaniel Bristot de Oliveira
2541e88ed227SDaniel Bristot de Oliveira /* This is the wakeup from this cycle */
2542e88ed227SDaniel Bristot de Oliveira now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer));
2543e88ed227SDaniel Bristot de Oliveira diff = now - tlat->abs_period;
2544e88ed227SDaniel Bristot de Oliveira
2545e88ed227SDaniel Bristot de Oliveira /*
2546e88ed227SDaniel Bristot de Oliveira * it was not a timer firing, but some other signal?
2547e88ed227SDaniel Bristot de Oliveira */
2548e88ed227SDaniel Bristot de Oliveira if (diff < 0)
2549e88ed227SDaniel Bristot de Oliveira goto out;
2550e88ed227SDaniel Bristot de Oliveira
2551e88ed227SDaniel Bristot de Oliveira s.seqnum = tlat->count;
2552e88ed227SDaniel Bristot de Oliveira s.timer_latency = diff;
2553e88ed227SDaniel Bristot de Oliveira s.context = THREAD_CONTEXT;
2554e88ed227SDaniel Bristot de Oliveira
2555e88ed227SDaniel Bristot de Oliveira trace_timerlat_sample(&s);
2556e88ed227SDaniel Bristot de Oliveira
2557e88ed227SDaniel Bristot de Oliveira if (osnoise_data.stop_tracing_total) {
2558e88ed227SDaniel Bristot de Oliveira if (time_to_us(diff) >= osnoise_data.stop_tracing_total) {
2559e88ed227SDaniel Bristot de Oliveira timerlat_dump_stack(time_to_us(diff));
2560e88ed227SDaniel Bristot de Oliveira notify_new_max_latency(diff);
2561e88ed227SDaniel Bristot de Oliveira osnoise_stop_tracing();
2562e88ed227SDaniel Bristot de Oliveira }
2563e88ed227SDaniel Bristot de Oliveira }
2564e88ed227SDaniel Bristot de Oliveira
2565e88ed227SDaniel Bristot de Oliveira out:
2566e88ed227SDaniel Bristot de Oliveira migrate_enable();
2567e88ed227SDaniel Bristot de Oliveira return 0;
2568e88ed227SDaniel Bristot de Oliveira }
2569e88ed227SDaniel Bristot de Oliveira
timerlat_fd_release(struct inode * inode,struct file * file)2570e88ed227SDaniel Bristot de Oliveira static int timerlat_fd_release(struct inode *inode, struct file *file)
2571e88ed227SDaniel Bristot de Oliveira {
2572e88ed227SDaniel Bristot de Oliveira struct osnoise_variables *osn_var;
2573e88ed227SDaniel Bristot de Oliveira struct timerlat_variables *tlat_var;
2574e88ed227SDaniel Bristot de Oliveira long cpu = (long) file->private_data;
2575e88ed227SDaniel Bristot de Oliveira
2576e88ed227SDaniel Bristot de Oliveira migrate_disable();
2577e88ed227SDaniel Bristot de Oliveira mutex_lock(&interface_lock);
2578e88ed227SDaniel Bristot de Oliveira
2579e88ed227SDaniel Bristot de Oliveira osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);
2580e88ed227SDaniel Bristot de Oliveira tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu);
2581e88ed227SDaniel Bristot de Oliveira
2582e88ed227SDaniel Bristot de Oliveira hrtimer_cancel(&tlat_var->timer);
2583e88ed227SDaniel Bristot de Oliveira memset(tlat_var, 0, sizeof(*tlat_var));
2584e88ed227SDaniel Bristot de Oliveira
2585e88ed227SDaniel Bristot de Oliveira osn_var->sampling = 0;
2586e88ed227SDaniel Bristot de Oliveira osn_var->pid = 0;
2587e88ed227SDaniel Bristot de Oliveira
2588e88ed227SDaniel Bristot de Oliveira /*
2589e88ed227SDaniel Bristot de Oliveira * We are leaving, not being stopped... see stop_kthread();
2590e88ed227SDaniel Bristot de Oliveira */
2591e88ed227SDaniel Bristot de Oliveira if (osn_var->kthread) {
2592e88ed227SDaniel Bristot de Oliveira put_task_struct(osn_var->kthread);
2593e88ed227SDaniel Bristot de Oliveira osn_var->kthread = NULL;
2594e88ed227SDaniel Bristot de Oliveira }
2595e88ed227SDaniel Bristot de Oliveira
2596e88ed227SDaniel Bristot de Oliveira mutex_unlock(&interface_lock);
2597e88ed227SDaniel Bristot de Oliveira migrate_enable();
2598e88ed227SDaniel Bristot de Oliveira return 0;
2599e88ed227SDaniel Bristot de Oliveira }
2600e88ed227SDaniel Bristot de Oliveira #endif
2601e88ed227SDaniel Bristot de Oliveira
2602bce29ac9SDaniel Bristot de Oliveira /*
2603bce29ac9SDaniel Bristot de Oliveira * osnoise/runtime_us: cannot be greater than the period.
2604bce29ac9SDaniel Bristot de Oliveira */
2605bce29ac9SDaniel Bristot de Oliveira static struct trace_min_max_param osnoise_runtime = {
2606bce29ac9SDaniel Bristot de Oliveira .lock = &interface_lock,
2607bce29ac9SDaniel Bristot de Oliveira .val = &osnoise_data.sample_runtime,
2608bce29ac9SDaniel Bristot de Oliveira .max = &osnoise_data.sample_period,
2609bce29ac9SDaniel Bristot de Oliveira .min = NULL,
2610bce29ac9SDaniel Bristot de Oliveira };
2611bce29ac9SDaniel Bristot de Oliveira
2612bce29ac9SDaniel Bristot de Oliveira /*
2613bce29ac9SDaniel Bristot de Oliveira * osnoise/period_us: cannot be smaller than the runtime.
2614bce29ac9SDaniel Bristot de Oliveira */
2615bce29ac9SDaniel Bristot de Oliveira static struct trace_min_max_param osnoise_period = {
2616bce29ac9SDaniel Bristot de Oliveira .lock = &interface_lock,
2617bce29ac9SDaniel Bristot de Oliveira .val = &osnoise_data.sample_period,
2618bce29ac9SDaniel Bristot de Oliveira .max = NULL,
2619bce29ac9SDaniel Bristot de Oliveira .min = &osnoise_data.sample_runtime,
2620bce29ac9SDaniel Bristot de Oliveira };
2621bce29ac9SDaniel Bristot de Oliveira
2622bce29ac9SDaniel Bristot de Oliveira /*
2623bce29ac9SDaniel Bristot de Oliveira * osnoise/stop_tracing_us: no limit.
2624bce29ac9SDaniel Bristot de Oliveira */
2625bce29ac9SDaniel Bristot de Oliveira static struct trace_min_max_param osnoise_stop_tracing_in = {
2626bce29ac9SDaniel Bristot de Oliveira .lock = &interface_lock,
2627bce29ac9SDaniel Bristot de Oliveira .val = &osnoise_data.stop_tracing,
2628bce29ac9SDaniel Bristot de Oliveira .max = NULL,
2629bce29ac9SDaniel Bristot de Oliveira .min = NULL,
2630bce29ac9SDaniel Bristot de Oliveira };
2631bce29ac9SDaniel Bristot de Oliveira
2632bce29ac9SDaniel Bristot de Oliveira /*
2633bce29ac9SDaniel Bristot de Oliveira * osnoise/stop_tracing_total_us: no limit.
2634bce29ac9SDaniel Bristot de Oliveira */
2635bce29ac9SDaniel Bristot de Oliveira static struct trace_min_max_param osnoise_stop_tracing_total = {
2636bce29ac9SDaniel Bristot de Oliveira .lock = &interface_lock,
2637bce29ac9SDaniel Bristot de Oliveira .val = &osnoise_data.stop_tracing_total,
2638bce29ac9SDaniel Bristot de Oliveira .max = NULL,
2639bce29ac9SDaniel Bristot de Oliveira .min = NULL,
2640bce29ac9SDaniel Bristot de Oliveira };
2641bce29ac9SDaniel Bristot de Oliveira
2642a955d7eaSDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
2643a955d7eaSDaniel Bristot de Oliveira /*
2644a955d7eaSDaniel Bristot de Oliveira * osnoise/print_stack: print the stacktrace of the IRQ handler if the total
2645a955d7eaSDaniel Bristot de Oliveira * latency is higher than val.
2646a955d7eaSDaniel Bristot de Oliveira */
2647a955d7eaSDaniel Bristot de Oliveira static struct trace_min_max_param osnoise_print_stack = {
2648a955d7eaSDaniel Bristot de Oliveira .lock = &interface_lock,
2649a955d7eaSDaniel Bristot de Oliveira .val = &osnoise_data.print_stack,
2650a955d7eaSDaniel Bristot de Oliveira .max = NULL,
2651a955d7eaSDaniel Bristot de Oliveira .min = NULL,
2652a955d7eaSDaniel Bristot de Oliveira };
2653a955d7eaSDaniel Bristot de Oliveira
2654a955d7eaSDaniel Bristot de Oliveira /*
2655a955d7eaSDaniel Bristot de Oliveira * osnoise/timerlat_period: min 100 us, max 1 s
2656a955d7eaSDaniel Bristot de Oliveira */
26577a025e06STom Rix static u64 timerlat_min_period = 100;
26587a025e06STom Rix static u64 timerlat_max_period = 1000000;
2659a955d7eaSDaniel Bristot de Oliveira static struct trace_min_max_param timerlat_period = {
2660a955d7eaSDaniel Bristot de Oliveira .lock = &interface_lock,
2661a955d7eaSDaniel Bristot de Oliveira .val = &osnoise_data.timerlat_period,
2662a955d7eaSDaniel Bristot de Oliveira .max = &timerlat_max_period,
2663a955d7eaSDaniel Bristot de Oliveira .min = &timerlat_min_period,
2664a955d7eaSDaniel Bristot de Oliveira };
2665e88ed227SDaniel Bristot de Oliveira
2666e88ed227SDaniel Bristot de Oliveira static const struct file_operations timerlat_fd_fops = {
2667e88ed227SDaniel Bristot de Oliveira .open = timerlat_fd_open,
2668e88ed227SDaniel Bristot de Oliveira .read = timerlat_fd_read,
2669e88ed227SDaniel Bristot de Oliveira .release = timerlat_fd_release,
2670e88ed227SDaniel Bristot de Oliveira .llseek = generic_file_llseek,
2671e88ed227SDaniel Bristot de Oliveira };
2672a955d7eaSDaniel Bristot de Oliveira #endif
2673a955d7eaSDaniel Bristot de Oliveira
2674bce29ac9SDaniel Bristot de Oliveira static const struct file_operations cpus_fops = {
2675bce29ac9SDaniel Bristot de Oliveira .open = tracing_open_generic,
2676bce29ac9SDaniel Bristot de Oliveira .read = osnoise_cpus_read,
2677bce29ac9SDaniel Bristot de Oliveira .write = osnoise_cpus_write,
2678bce29ac9SDaniel Bristot de Oliveira .llseek = generic_file_llseek,
2679bce29ac9SDaniel Bristot de Oliveira };
2680bce29ac9SDaniel Bristot de Oliveira
2681b179d48bSDaniel Bristot de Oliveira static const struct file_operations osnoise_options_fops = {
2682b179d48bSDaniel Bristot de Oliveira .open = osnoise_options_open,
2683b179d48bSDaniel Bristot de Oliveira .read = seq_read,
2684b179d48bSDaniel Bristot de Oliveira .llseek = seq_lseek,
2685b179d48bSDaniel Bristot de Oliveira .release = seq_release,
2686b179d48bSDaniel Bristot de Oliveira .write = osnoise_options_write
2687b179d48bSDaniel Bristot de Oliveira };
2688b179d48bSDaniel Bristot de Oliveira
2689ccb67544SDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
2690b14f4568SDaniel Bristot de Oliveira #ifdef CONFIG_STACKTRACE
init_timerlat_stack_tracefs(struct dentry * top_dir)2691b14f4568SDaniel Bristot de Oliveira static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2692b14f4568SDaniel Bristot de Oliveira {
2693b14f4568SDaniel Bristot de Oliveira struct dentry *tmp;
2694b14f4568SDaniel Bristot de Oliveira
2695b14f4568SDaniel Bristot de Oliveira tmp = tracefs_create_file("print_stack", TRACE_MODE_WRITE, top_dir,
2696b14f4568SDaniel Bristot de Oliveira &osnoise_print_stack, &trace_min_max_fops);
2697b14f4568SDaniel Bristot de Oliveira if (!tmp)
2698b14f4568SDaniel Bristot de Oliveira return -ENOMEM;
2699b14f4568SDaniel Bristot de Oliveira
2700b14f4568SDaniel Bristot de Oliveira return 0;
2701b14f4568SDaniel Bristot de Oliveira }
2702b14f4568SDaniel Bristot de Oliveira #else /* CONFIG_STACKTRACE */
init_timerlat_stack_tracefs(struct dentry * top_dir)2703b14f4568SDaniel Bristot de Oliveira static int init_timerlat_stack_tracefs(struct dentry *top_dir)
2704b14f4568SDaniel Bristot de Oliveira {
2705b14f4568SDaniel Bristot de Oliveira return 0;
2706b14f4568SDaniel Bristot de Oliveira }
2707b14f4568SDaniel Bristot de Oliveira #endif /* CONFIG_STACKTRACE */
2708b14f4568SDaniel Bristot de Oliveira
osnoise_create_cpu_timerlat_fd(struct dentry * top_dir)2709e88ed227SDaniel Bristot de Oliveira static int osnoise_create_cpu_timerlat_fd(struct dentry *top_dir)
2710e88ed227SDaniel Bristot de Oliveira {
2711e88ed227SDaniel Bristot de Oliveira struct dentry *timerlat_fd;
2712e88ed227SDaniel Bristot de Oliveira struct dentry *per_cpu;
2713e88ed227SDaniel Bristot de Oliveira struct dentry *cpu_dir;
2714e88ed227SDaniel Bristot de Oliveira char cpu_str[30]; /* see trace.c: tracing_init_tracefs_percpu() */
2715e88ed227SDaniel Bristot de Oliveira long cpu;
2716e88ed227SDaniel Bristot de Oliveira
2717e88ed227SDaniel Bristot de Oliveira /*
2718e88ed227SDaniel Bristot de Oliveira * Why not using tracing instance per_cpu/ dir?
2719e88ed227SDaniel Bristot de Oliveira *
2720e88ed227SDaniel Bristot de Oliveira * Because osnoise/timerlat have a single workload, having
2721e88ed227SDaniel Bristot de Oliveira * multiple files like these are wast of memory.
2722e88ed227SDaniel Bristot de Oliveira */
2723e88ed227SDaniel Bristot de Oliveira per_cpu = tracefs_create_dir("per_cpu", top_dir);
2724e88ed227SDaniel Bristot de Oliveira if (!per_cpu)
2725e88ed227SDaniel Bristot de Oliveira return -ENOMEM;
2726e88ed227SDaniel Bristot de Oliveira
2727e88ed227SDaniel Bristot de Oliveira for_each_possible_cpu(cpu) {
2728e88ed227SDaniel Bristot de Oliveira snprintf(cpu_str, 30, "cpu%ld", cpu);
2729e88ed227SDaniel Bristot de Oliveira cpu_dir = tracefs_create_dir(cpu_str, per_cpu);
2730e88ed227SDaniel Bristot de Oliveira if (!cpu_dir)
2731e88ed227SDaniel Bristot de Oliveira goto out_clean;
2732e88ed227SDaniel Bristot de Oliveira
2733e88ed227SDaniel Bristot de Oliveira timerlat_fd = trace_create_file("timerlat_fd", TRACE_MODE_READ,
2734e88ed227SDaniel Bristot de Oliveira cpu_dir, NULL, &timerlat_fd_fops);
2735e88ed227SDaniel Bristot de Oliveira if (!timerlat_fd)
2736e88ed227SDaniel Bristot de Oliveira goto out_clean;
2737e88ed227SDaniel Bristot de Oliveira
2738e88ed227SDaniel Bristot de Oliveira /* Record the CPU */
2739e88ed227SDaniel Bristot de Oliveira d_inode(timerlat_fd)->i_cdev = (void *)(cpu);
2740e88ed227SDaniel Bristot de Oliveira }
2741e88ed227SDaniel Bristot de Oliveira
2742e88ed227SDaniel Bristot de Oliveira return 0;
2743e88ed227SDaniel Bristot de Oliveira
2744e88ed227SDaniel Bristot de Oliveira out_clean:
2745e88ed227SDaniel Bristot de Oliveira tracefs_remove(per_cpu);
2746e88ed227SDaniel Bristot de Oliveira return -ENOMEM;
2747e88ed227SDaniel Bristot de Oliveira }
2748e88ed227SDaniel Bristot de Oliveira
2749ccb67544SDaniel Bristot de Oliveira /*
2750ccb67544SDaniel Bristot de Oliveira * init_timerlat_tracefs - A function to initialize the timerlat interface files
2751ccb67544SDaniel Bristot de Oliveira */
init_timerlat_tracefs(struct dentry * top_dir)2752ccb67544SDaniel Bristot de Oliveira static int init_timerlat_tracefs(struct dentry *top_dir)
2753ccb67544SDaniel Bristot de Oliveira {
2754ccb67544SDaniel Bristot de Oliveira struct dentry *tmp;
2755e88ed227SDaniel Bristot de Oliveira int retval;
2756ccb67544SDaniel Bristot de Oliveira
2757ccb67544SDaniel Bristot de Oliveira tmp = tracefs_create_file("timerlat_period_us", TRACE_MODE_WRITE, top_dir,
2758ccb67544SDaniel Bristot de Oliveira &timerlat_period, &trace_min_max_fops);
2759ccb67544SDaniel Bristot de Oliveira if (!tmp)
2760ccb67544SDaniel Bristot de Oliveira return -ENOMEM;
2761ccb67544SDaniel Bristot de Oliveira
2762e88ed227SDaniel Bristot de Oliveira retval = osnoise_create_cpu_timerlat_fd(top_dir);
2763e88ed227SDaniel Bristot de Oliveira if (retval)
2764e88ed227SDaniel Bristot de Oliveira return retval;
2765e88ed227SDaniel Bristot de Oliveira
2766b14f4568SDaniel Bristot de Oliveira return init_timerlat_stack_tracefs(top_dir);
2767ccb67544SDaniel Bristot de Oliveira }
2768ccb67544SDaniel Bristot de Oliveira #else /* CONFIG_TIMERLAT_TRACER */
init_timerlat_tracefs(struct dentry * top_dir)2769ccb67544SDaniel Bristot de Oliveira static int init_timerlat_tracefs(struct dentry *top_dir)
2770ccb67544SDaniel Bristot de Oliveira {
2771ccb67544SDaniel Bristot de Oliveira return 0;
2772ccb67544SDaniel Bristot de Oliveira }
2773ccb67544SDaniel Bristot de Oliveira #endif /* CONFIG_TIMERLAT_TRACER */
2774ccb67544SDaniel Bristot de Oliveira
2775bce29ac9SDaniel Bristot de Oliveira /*
2776bce29ac9SDaniel Bristot de Oliveira * init_tracefs - A function to initialize the tracefs interface files
2777bce29ac9SDaniel Bristot de Oliveira *
2778a955d7eaSDaniel Bristot de Oliveira * This function creates entries in tracefs for "osnoise" and "timerlat".
2779a955d7eaSDaniel Bristot de Oliveira * It creates these directories in the tracing directory, and within that
2780a955d7eaSDaniel Bristot de Oliveira * directory the use can change and view the configs.
2781bce29ac9SDaniel Bristot de Oliveira */
init_tracefs(void)2782bce29ac9SDaniel Bristot de Oliveira static int init_tracefs(void)
2783bce29ac9SDaniel Bristot de Oliveira {
2784bce29ac9SDaniel Bristot de Oliveira struct dentry *top_dir;
2785bce29ac9SDaniel Bristot de Oliveira struct dentry *tmp;
2786bce29ac9SDaniel Bristot de Oliveira int ret;
2787bce29ac9SDaniel Bristot de Oliveira
2788bce29ac9SDaniel Bristot de Oliveira ret = tracing_init_dentry();
2789bce29ac9SDaniel Bristot de Oliveira if (ret)
2790bce29ac9SDaniel Bristot de Oliveira return -ENOMEM;
2791bce29ac9SDaniel Bristot de Oliveira
2792bce29ac9SDaniel Bristot de Oliveira top_dir = tracefs_create_dir("osnoise", NULL);
2793bce29ac9SDaniel Bristot de Oliveira if (!top_dir)
2794a955d7eaSDaniel Bristot de Oliveira return 0;
2795bce29ac9SDaniel Bristot de Oliveira
279621ccc9cdSSteven Rostedt (VMware) tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir,
2797bce29ac9SDaniel Bristot de Oliveira &osnoise_period, &trace_min_max_fops);
2798bce29ac9SDaniel Bristot de Oliveira if (!tmp)
2799bce29ac9SDaniel Bristot de Oliveira goto err;
2800bce29ac9SDaniel Bristot de Oliveira
280121ccc9cdSSteven Rostedt (VMware) tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir,
2802bce29ac9SDaniel Bristot de Oliveira &osnoise_runtime, &trace_min_max_fops);
2803bce29ac9SDaniel Bristot de Oliveira if (!tmp)
2804bce29ac9SDaniel Bristot de Oliveira goto err;
2805bce29ac9SDaniel Bristot de Oliveira
280621ccc9cdSSteven Rostedt (VMware) tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir,
2807bce29ac9SDaniel Bristot de Oliveira &osnoise_stop_tracing_in, &trace_min_max_fops);
2808bce29ac9SDaniel Bristot de Oliveira if (!tmp)
2809bce29ac9SDaniel Bristot de Oliveira goto err;
2810bce29ac9SDaniel Bristot de Oliveira
281121ccc9cdSSteven Rostedt (VMware) tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir,
2812bce29ac9SDaniel Bristot de Oliveira &osnoise_stop_tracing_total, &trace_min_max_fops);
2813bce29ac9SDaniel Bristot de Oliveira if (!tmp)
2814bce29ac9SDaniel Bristot de Oliveira goto err;
2815bce29ac9SDaniel Bristot de Oliveira
281621ccc9cdSSteven Rostedt (VMware) tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops);
2817bce29ac9SDaniel Bristot de Oliveira if (!tmp)
2818bce29ac9SDaniel Bristot de Oliveira goto err;
2819a955d7eaSDaniel Bristot de Oliveira
2820b179d48bSDaniel Bristot de Oliveira tmp = trace_create_file("options", TRACE_MODE_WRITE, top_dir, NULL,
2821b179d48bSDaniel Bristot de Oliveira &osnoise_options_fops);
2822b179d48bSDaniel Bristot de Oliveira if (!tmp)
2823b179d48bSDaniel Bristot de Oliveira goto err;
2824b179d48bSDaniel Bristot de Oliveira
2825ccb67544SDaniel Bristot de Oliveira ret = init_timerlat_tracefs(top_dir);
2826ccb67544SDaniel Bristot de Oliveira if (ret)
2827a955d7eaSDaniel Bristot de Oliveira goto err;
2828bce29ac9SDaniel Bristot de Oliveira
2829bce29ac9SDaniel Bristot de Oliveira return 0;
2830bce29ac9SDaniel Bristot de Oliveira
2831bce29ac9SDaniel Bristot de Oliveira err:
2832bce29ac9SDaniel Bristot de Oliveira tracefs_remove(top_dir);
2833bce29ac9SDaniel Bristot de Oliveira return -ENOMEM;
2834bce29ac9SDaniel Bristot de Oliveira }
2835bce29ac9SDaniel Bristot de Oliveira
osnoise_hook_events(void)2836bce29ac9SDaniel Bristot de Oliveira static int osnoise_hook_events(void)
2837bce29ac9SDaniel Bristot de Oliveira {
2838bce29ac9SDaniel Bristot de Oliveira int retval;
2839bce29ac9SDaniel Bristot de Oliveira
2840bce29ac9SDaniel Bristot de Oliveira /*
2841bce29ac9SDaniel Bristot de Oliveira * Trace is already hooked, we are re-enabling from
2842bce29ac9SDaniel Bristot de Oliveira * a stop_tracing_*.
2843bce29ac9SDaniel Bristot de Oliveira */
2844bce29ac9SDaniel Bristot de Oliveira if (trace_osnoise_callback_enabled)
2845bce29ac9SDaniel Bristot de Oliveira return 0;
2846bce29ac9SDaniel Bristot de Oliveira
2847bce29ac9SDaniel Bristot de Oliveira retval = hook_irq_events();
2848bce29ac9SDaniel Bristot de Oliveira if (retval)
2849bce29ac9SDaniel Bristot de Oliveira return -EINVAL;
2850bce29ac9SDaniel Bristot de Oliveira
2851bce29ac9SDaniel Bristot de Oliveira retval = hook_softirq_events();
2852bce29ac9SDaniel Bristot de Oliveira if (retval)
2853bce29ac9SDaniel Bristot de Oliveira goto out_unhook_irq;
2854bce29ac9SDaniel Bristot de Oliveira
2855bce29ac9SDaniel Bristot de Oliveira retval = hook_thread_events();
2856bce29ac9SDaniel Bristot de Oliveira /*
2857bce29ac9SDaniel Bristot de Oliveira * All fine!
2858bce29ac9SDaniel Bristot de Oliveira */
2859bce29ac9SDaniel Bristot de Oliveira if (!retval)
2860bce29ac9SDaniel Bristot de Oliveira return 0;
2861bce29ac9SDaniel Bristot de Oliveira
2862bce29ac9SDaniel Bristot de Oliveira unhook_softirq_events();
2863bce29ac9SDaniel Bristot de Oliveira out_unhook_irq:
2864bce29ac9SDaniel Bristot de Oliveira unhook_irq_events();
2865bce29ac9SDaniel Bristot de Oliveira return -EINVAL;
2866bce29ac9SDaniel Bristot de Oliveira }
2867bce29ac9SDaniel Bristot de Oliveira
osnoise_unhook_events(void)28680878355bSNikita Yushchenko static void osnoise_unhook_events(void)
28690878355bSNikita Yushchenko {
28700878355bSNikita Yushchenko unhook_thread_events();
28710878355bSNikita Yushchenko unhook_softirq_events();
28720878355bSNikita Yushchenko unhook_irq_events();
28730878355bSNikita Yushchenko }
28740878355bSNikita Yushchenko
287515ca4bdbSDaniel Bristot de Oliveira /*
287615ca4bdbSDaniel Bristot de Oliveira * osnoise_workload_start - start the workload and hook to events
287715ca4bdbSDaniel Bristot de Oliveira */
osnoise_workload_start(void)287815ca4bdbSDaniel Bristot de Oliveira static int osnoise_workload_start(void)
2879bce29ac9SDaniel Bristot de Oliveira {
2880bce29ac9SDaniel Bristot de Oliveira int retval;
2881bce29ac9SDaniel Bristot de Oliveira
28822fac8d64SDaniel Bristot de Oliveira /*
28832fac8d64SDaniel Bristot de Oliveira * Instances need to be registered after calling workload
28842fac8d64SDaniel Bristot de Oliveira * start. Hence, if there is already an instance, the
28852fac8d64SDaniel Bristot de Oliveira * workload was already registered. Otherwise, this
28862fac8d64SDaniel Bristot de Oliveira * code is on the way to register the first instance,
28872fac8d64SDaniel Bristot de Oliveira * and the workload will start.
28882fac8d64SDaniel Bristot de Oliveira */
28892fac8d64SDaniel Bristot de Oliveira if (osnoise_has_registered_instances())
28902fac8d64SDaniel Bristot de Oliveira return 0;
28912fac8d64SDaniel Bristot de Oliveira
2892bce29ac9SDaniel Bristot de Oliveira osn_var_reset_all();
2893bce29ac9SDaniel Bristot de Oliveira
2894bce29ac9SDaniel Bristot de Oliveira retval = osnoise_hook_events();
2895bce29ac9SDaniel Bristot de Oliveira if (retval)
2896a955d7eaSDaniel Bristot de Oliveira return retval;
2897c3b6343cSDaniel Bristot de Oliveira
2898bce29ac9SDaniel Bristot de Oliveira /*
2899c3b6343cSDaniel Bristot de Oliveira * Make sure that ftrace_nmi_enter/exit() see reset values
2900c3b6343cSDaniel Bristot de Oliveira * before enabling trace_osnoise_callback_enabled.
2901bce29ac9SDaniel Bristot de Oliveira */
2902bce29ac9SDaniel Bristot de Oliveira barrier();
2903bce29ac9SDaniel Bristot de Oliveira trace_osnoise_callback_enabled = true;
2904bce29ac9SDaniel Bristot de Oliveira
290515ca4bdbSDaniel Bristot de Oliveira retval = start_per_cpu_kthreads();
2906a955d7eaSDaniel Bristot de Oliveira if (retval) {
29070878355bSNikita Yushchenko trace_osnoise_callback_enabled = false;
29080878355bSNikita Yushchenko /*
29090878355bSNikita Yushchenko * Make sure that ftrace_nmi_enter/exit() see
29100878355bSNikita Yushchenko * trace_osnoise_callback_enabled as false before continuing.
29110878355bSNikita Yushchenko */
29120878355bSNikita Yushchenko barrier();
29130878355bSNikita Yushchenko
29140878355bSNikita Yushchenko osnoise_unhook_events();
2915a955d7eaSDaniel Bristot de Oliveira return retval;
2916a955d7eaSDaniel Bristot de Oliveira }
2917a955d7eaSDaniel Bristot de Oliveira
2918a955d7eaSDaniel Bristot de Oliveira return 0;
2919a955d7eaSDaniel Bristot de Oliveira }
2920a955d7eaSDaniel Bristot de Oliveira
292115ca4bdbSDaniel Bristot de Oliveira /*
292215ca4bdbSDaniel Bristot de Oliveira * osnoise_workload_stop - stop the workload and unhook the events
292315ca4bdbSDaniel Bristot de Oliveira */
osnoise_workload_stop(void)292415ca4bdbSDaniel Bristot de Oliveira static void osnoise_workload_stop(void)
2925bce29ac9SDaniel Bristot de Oliveira {
29262fac8d64SDaniel Bristot de Oliveira /*
29272fac8d64SDaniel Bristot de Oliveira * Instances need to be unregistered before calling
29282fac8d64SDaniel Bristot de Oliveira * stop. Hence, if there is a registered instance, more
29292fac8d64SDaniel Bristot de Oliveira * than one instance is running, and the workload will not
29302fac8d64SDaniel Bristot de Oliveira * yet stop. Otherwise, this code is on the way to disable
29312fac8d64SDaniel Bristot de Oliveira * the last instance, and the workload can stop.
29322fac8d64SDaniel Bristot de Oliveira */
2933dae18134SDaniel Bristot de Oliveira if (osnoise_has_registered_instances())
2934bce29ac9SDaniel Bristot de Oliveira return;
2935bce29ac9SDaniel Bristot de Oliveira
2936f0cfe17bSDaniel Bristot de Oliveira /*
2937f0cfe17bSDaniel Bristot de Oliveira * If callbacks were already disabled in a previous stop
2938f0cfe17bSDaniel Bristot de Oliveira * call, there is no need to disable then again.
2939f0cfe17bSDaniel Bristot de Oliveira *
2940f0cfe17bSDaniel Bristot de Oliveira * For instance, this happens when tracing is stopped via:
2941f0cfe17bSDaniel Bristot de Oliveira * echo 0 > tracing_on
2942f0cfe17bSDaniel Bristot de Oliveira * echo nop > current_tracer.
2943f0cfe17bSDaniel Bristot de Oliveira */
2944f0cfe17bSDaniel Bristot de Oliveira if (!trace_osnoise_callback_enabled)
2945f0cfe17bSDaniel Bristot de Oliveira return;
2946f0cfe17bSDaniel Bristot de Oliveira
2947bce29ac9SDaniel Bristot de Oliveira trace_osnoise_callback_enabled = false;
2948c3b6343cSDaniel Bristot de Oliveira /*
2949c3b6343cSDaniel Bristot de Oliveira * Make sure that ftrace_nmi_enter/exit() see
2950c3b6343cSDaniel Bristot de Oliveira * trace_osnoise_callback_enabled as false before continuing.
2951c3b6343cSDaniel Bristot de Oliveira */
2952bce29ac9SDaniel Bristot de Oliveira barrier();
2953bce29ac9SDaniel Bristot de Oliveira
2954bce29ac9SDaniel Bristot de Oliveira stop_per_cpu_kthreads();
2955bce29ac9SDaniel Bristot de Oliveira
29560878355bSNikita Yushchenko osnoise_unhook_events();
2957bce29ac9SDaniel Bristot de Oliveira }
2958bce29ac9SDaniel Bristot de Oliveira
osnoise_tracer_start(struct trace_array * tr)295915ca4bdbSDaniel Bristot de Oliveira static void osnoise_tracer_start(struct trace_array *tr)
296015ca4bdbSDaniel Bristot de Oliveira {
296115ca4bdbSDaniel Bristot de Oliveira int retval;
296215ca4bdbSDaniel Bristot de Oliveira
29632fac8d64SDaniel Bristot de Oliveira /*
29642fac8d64SDaniel Bristot de Oliveira * If the instance is already registered, there is no need to
29652fac8d64SDaniel Bristot de Oliveira * register it again.
29662fac8d64SDaniel Bristot de Oliveira */
29672fac8d64SDaniel Bristot de Oliveira if (osnoise_instance_registered(tr))
296815ca4bdbSDaniel Bristot de Oliveira return;
296915ca4bdbSDaniel Bristot de Oliveira
297015ca4bdbSDaniel Bristot de Oliveira retval = osnoise_workload_start();
297115ca4bdbSDaniel Bristot de Oliveira if (retval)
297215ca4bdbSDaniel Bristot de Oliveira pr_err(BANNER "Error starting osnoise tracer\n");
297315ca4bdbSDaniel Bristot de Oliveira
2974dae18134SDaniel Bristot de Oliveira osnoise_register_instance(tr);
297515ca4bdbSDaniel Bristot de Oliveira }
297615ca4bdbSDaniel Bristot de Oliveira
osnoise_tracer_stop(struct trace_array * tr)297715ca4bdbSDaniel Bristot de Oliveira static void osnoise_tracer_stop(struct trace_array *tr)
297815ca4bdbSDaniel Bristot de Oliveira {
2979dae18134SDaniel Bristot de Oliveira osnoise_unregister_instance(tr);
298015ca4bdbSDaniel Bristot de Oliveira osnoise_workload_stop();
298115ca4bdbSDaniel Bristot de Oliveira }
298215ca4bdbSDaniel Bristot de Oliveira
osnoise_tracer_init(struct trace_array * tr)2983bce29ac9SDaniel Bristot de Oliveira static int osnoise_tracer_init(struct trace_array *tr)
2984bce29ac9SDaniel Bristot de Oliveira {
29852fac8d64SDaniel Bristot de Oliveira /*
29862fac8d64SDaniel Bristot de Oliveira * Only allow osnoise tracer if timerlat tracer is not running
29872fac8d64SDaniel Bristot de Oliveira * already.
29882fac8d64SDaniel Bristot de Oliveira */
29892fac8d64SDaniel Bristot de Oliveira if (timerlat_enabled())
2990bce29ac9SDaniel Bristot de Oliveira return -EBUSY;
2991bce29ac9SDaniel Bristot de Oliveira
2992bce29ac9SDaniel Bristot de Oliveira tr->max_latency = 0;
2993bce29ac9SDaniel Bristot de Oliveira
2994bce29ac9SDaniel Bristot de Oliveira osnoise_tracer_start(tr);
2995bce29ac9SDaniel Bristot de Oliveira return 0;
2996bce29ac9SDaniel Bristot de Oliveira }
2997bce29ac9SDaniel Bristot de Oliveira
osnoise_tracer_reset(struct trace_array * tr)2998bce29ac9SDaniel Bristot de Oliveira static void osnoise_tracer_reset(struct trace_array *tr)
2999bce29ac9SDaniel Bristot de Oliveira {
3000bce29ac9SDaniel Bristot de Oliveira osnoise_tracer_stop(tr);
3001bce29ac9SDaniel Bristot de Oliveira }
3002bce29ac9SDaniel Bristot de Oliveira
3003bce29ac9SDaniel Bristot de Oliveira static struct tracer osnoise_tracer __read_mostly = {
3004bce29ac9SDaniel Bristot de Oliveira .name = "osnoise",
3005bce29ac9SDaniel Bristot de Oliveira .init = osnoise_tracer_init,
3006bce29ac9SDaniel Bristot de Oliveira .reset = osnoise_tracer_reset,
3007bce29ac9SDaniel Bristot de Oliveira .start = osnoise_tracer_start,
3008bce29ac9SDaniel Bristot de Oliveira .stop = osnoise_tracer_stop,
3009bce29ac9SDaniel Bristot de Oliveira .print_header = print_osnoise_headers,
3010bce29ac9SDaniel Bristot de Oliveira .allow_instances = true,
3011bce29ac9SDaniel Bristot de Oliveira };
3012bce29ac9SDaniel Bristot de Oliveira
3013a955d7eaSDaniel Bristot de Oliveira #ifdef CONFIG_TIMERLAT_TRACER
timerlat_tracer_start(struct trace_array * tr)3014a955d7eaSDaniel Bristot de Oliveira static void timerlat_tracer_start(struct trace_array *tr)
3015a955d7eaSDaniel Bristot de Oliveira {
3016a955d7eaSDaniel Bristot de Oliveira int retval;
3017a955d7eaSDaniel Bristot de Oliveira
30182fac8d64SDaniel Bristot de Oliveira /*
30192fac8d64SDaniel Bristot de Oliveira * If the instance is already registered, there is no need to
30202fac8d64SDaniel Bristot de Oliveira * register it again.
30212fac8d64SDaniel Bristot de Oliveira */
30222fac8d64SDaniel Bristot de Oliveira if (osnoise_instance_registered(tr))
3023a955d7eaSDaniel Bristot de Oliveira return;
3024a955d7eaSDaniel Bristot de Oliveira
302515ca4bdbSDaniel Bristot de Oliveira retval = osnoise_workload_start();
3026a955d7eaSDaniel Bristot de Oliveira if (retval)
30272fac8d64SDaniel Bristot de Oliveira pr_err(BANNER "Error starting timerlat tracer\n");
3028a955d7eaSDaniel Bristot de Oliveira
3029dae18134SDaniel Bristot de Oliveira osnoise_register_instance(tr);
3030dae18134SDaniel Bristot de Oliveira
3031a955d7eaSDaniel Bristot de Oliveira return;
3032a955d7eaSDaniel Bristot de Oliveira }
3033a955d7eaSDaniel Bristot de Oliveira
timerlat_tracer_stop(struct trace_array * tr)3034a955d7eaSDaniel Bristot de Oliveira static void timerlat_tracer_stop(struct trace_array *tr)
3035a955d7eaSDaniel Bristot de Oliveira {
3036a955d7eaSDaniel Bristot de Oliveira int cpu;
3037a955d7eaSDaniel Bristot de Oliveira
30382fac8d64SDaniel Bristot de Oliveira osnoise_unregister_instance(tr);
3039a955d7eaSDaniel Bristot de Oliveira
30402fac8d64SDaniel Bristot de Oliveira /*
30412fac8d64SDaniel Bristot de Oliveira * Instruct the threads to stop only if this is the last instance.
30422fac8d64SDaniel Bristot de Oliveira */
30432fac8d64SDaniel Bristot de Oliveira if (!osnoise_has_registered_instances()) {
3044a955d7eaSDaniel Bristot de Oliveira for_each_online_cpu(cpu)
3045a955d7eaSDaniel Bristot de Oliveira per_cpu(per_cpu_osnoise_var, cpu).sampling = 0;
30462fac8d64SDaniel Bristot de Oliveira }
3047a955d7eaSDaniel Bristot de Oliveira
30482fac8d64SDaniel Bristot de Oliveira osnoise_workload_stop();
3049a955d7eaSDaniel Bristot de Oliveira }
3050a955d7eaSDaniel Bristot de Oliveira
timerlat_tracer_init(struct trace_array * tr)3051a955d7eaSDaniel Bristot de Oliveira static int timerlat_tracer_init(struct trace_array *tr)
3052a955d7eaSDaniel Bristot de Oliveira {
30532fac8d64SDaniel Bristot de Oliveira /*
30542fac8d64SDaniel Bristot de Oliveira * Only allow timerlat tracer if osnoise tracer is not running already.
30552fac8d64SDaniel Bristot de Oliveira */
30562fac8d64SDaniel Bristot de Oliveira if (osnoise_has_registered_instances() && !osnoise_data.timerlat_tracer)
3057a955d7eaSDaniel Bristot de Oliveira return -EBUSY;
3058a955d7eaSDaniel Bristot de Oliveira
30592fac8d64SDaniel Bristot de Oliveira /*
30602fac8d64SDaniel Bristot de Oliveira * If this is the first instance, set timerlat_tracer to block
30612fac8d64SDaniel Bristot de Oliveira * osnoise tracer start.
30622fac8d64SDaniel Bristot de Oliveira */
30632fac8d64SDaniel Bristot de Oliveira if (!osnoise_has_registered_instances())
30642fac8d64SDaniel Bristot de Oliveira osnoise_data.timerlat_tracer = 1;
3065a955d7eaSDaniel Bristot de Oliveira
30662fac8d64SDaniel Bristot de Oliveira tr->max_latency = 0;
3067a955d7eaSDaniel Bristot de Oliveira timerlat_tracer_start(tr);
3068a955d7eaSDaniel Bristot de Oliveira
3069a955d7eaSDaniel Bristot de Oliveira return 0;
3070a955d7eaSDaniel Bristot de Oliveira }
3071a955d7eaSDaniel Bristot de Oliveira
timerlat_tracer_reset(struct trace_array * tr)3072a955d7eaSDaniel Bristot de Oliveira static void timerlat_tracer_reset(struct trace_array *tr)
3073a955d7eaSDaniel Bristot de Oliveira {
3074a955d7eaSDaniel Bristot de Oliveira timerlat_tracer_stop(tr);
30752fac8d64SDaniel Bristot de Oliveira
30762fac8d64SDaniel Bristot de Oliveira /*
30772fac8d64SDaniel Bristot de Oliveira * If this is the last instance, reset timerlat_tracer allowing
30782fac8d64SDaniel Bristot de Oliveira * osnoise to be started.
30792fac8d64SDaniel Bristot de Oliveira */
30802fac8d64SDaniel Bristot de Oliveira if (!osnoise_has_registered_instances())
30812fac8d64SDaniel Bristot de Oliveira osnoise_data.timerlat_tracer = 0;
3082a955d7eaSDaniel Bristot de Oliveira }
3083a955d7eaSDaniel Bristot de Oliveira
3084a955d7eaSDaniel Bristot de Oliveira static struct tracer timerlat_tracer __read_mostly = {
3085a955d7eaSDaniel Bristot de Oliveira .name = "timerlat",
3086a955d7eaSDaniel Bristot de Oliveira .init = timerlat_tracer_init,
3087a955d7eaSDaniel Bristot de Oliveira .reset = timerlat_tracer_reset,
3088a955d7eaSDaniel Bristot de Oliveira .start = timerlat_tracer_start,
3089a955d7eaSDaniel Bristot de Oliveira .stop = timerlat_tracer_stop,
3090a955d7eaSDaniel Bristot de Oliveira .print_header = print_timerlat_headers,
3091a955d7eaSDaniel Bristot de Oliveira .allow_instances = true,
3092a955d7eaSDaniel Bristot de Oliveira };
3093ccb67544SDaniel Bristot de Oliveira
init_timerlat_tracer(void)3094ccb67544SDaniel Bristot de Oliveira __init static int init_timerlat_tracer(void)
3095ccb67544SDaniel Bristot de Oliveira {
3096ccb67544SDaniel Bristot de Oliveira return register_tracer(&timerlat_tracer);
3097ccb67544SDaniel Bristot de Oliveira }
3098ccb67544SDaniel Bristot de Oliveira #else /* CONFIG_TIMERLAT_TRACER */
init_timerlat_tracer(void)3099ccb67544SDaniel Bristot de Oliveira __init static int init_timerlat_tracer(void)
3100ccb67544SDaniel Bristot de Oliveira {
3101ccb67544SDaniel Bristot de Oliveira return 0;
3102ccb67544SDaniel Bristot de Oliveira }
3103a955d7eaSDaniel Bristot de Oliveira #endif /* CONFIG_TIMERLAT_TRACER */
3104a955d7eaSDaniel Bristot de Oliveira
init_osnoise_tracer(void)3105bce29ac9SDaniel Bristot de Oliveira __init static int init_osnoise_tracer(void)
3106bce29ac9SDaniel Bristot de Oliveira {
3107bce29ac9SDaniel Bristot de Oliveira int ret;
3108bce29ac9SDaniel Bristot de Oliveira
3109bce29ac9SDaniel Bristot de Oliveira mutex_init(&interface_lock);
3110bce29ac9SDaniel Bristot de Oliveira
3111bce29ac9SDaniel Bristot de Oliveira cpumask_copy(&osnoise_cpumask, cpu_all_mask);
3112bce29ac9SDaniel Bristot de Oliveira
3113bce29ac9SDaniel Bristot de Oliveira ret = register_tracer(&osnoise_tracer);
3114a955d7eaSDaniel Bristot de Oliveira if (ret) {
3115a955d7eaSDaniel Bristot de Oliveira pr_err(BANNER "Error registering osnoise!\n");
3116bce29ac9SDaniel Bristot de Oliveira return ret;
3117a955d7eaSDaniel Bristot de Oliveira }
3118a955d7eaSDaniel Bristot de Oliveira
3119ccb67544SDaniel Bristot de Oliveira ret = init_timerlat_tracer();
3120a955d7eaSDaniel Bristot de Oliveira if (ret) {
3121ccb67544SDaniel Bristot de Oliveira pr_err(BANNER "Error registering timerlat!\n");
3122a955d7eaSDaniel Bristot de Oliveira return ret;
3123a955d7eaSDaniel Bristot de Oliveira }
3124ccb67544SDaniel Bristot de Oliveira
3125c8895e27SDaniel Bristot de Oliveira osnoise_init_hotplug_support();
3126bce29ac9SDaniel Bristot de Oliveira
3127dae18134SDaniel Bristot de Oliveira INIT_LIST_HEAD_RCU(&osnoise_instances);
3128dae18134SDaniel Bristot de Oliveira
3129bce29ac9SDaniel Bristot de Oliveira init_tracefs();
3130bce29ac9SDaniel Bristot de Oliveira
3131bce29ac9SDaniel Bristot de Oliveira return 0;
3132bce29ac9SDaniel Bristot de Oliveira }
3133bce29ac9SDaniel Bristot de Oliveira late_initcall(init_osnoise_tracer);
3134