1 // SPDX-License-Identifier: GPL-2.0
3  * trace_hwlat.c - A simple Hardware Latency detector.
18  * SMI mode (complete with special memory map), and the OS is unaware.
20  * Although certain hardware-inducing latencies are necessary (for example,
22  * and remote management) they can wreak havoc upon any OS-level performance
23  * guarantees toward low-latency, especially when the OS is not even made
27  * sampling the built-in CPU timer, looking for discontiguous readings.
31  *          environment requiring any kind of low-latency performance
34  * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. <jcm@redhat.com>
35  * Copyright (C) 2013-2016 Steven Rostedt, Red Hat, Inc. <srostedt@redhat.com>
59 static struct dentry *hwlat_thread_mode;	/* hwlat thread mode */
67 static char *thread_mode_str[] = { "none", "round-robin", "per-cpu" };
112 	int	thread_mode;		/* thread mode */
134 	struct trace_buffer *buffer = tr->array_buffer.buffer;  in trace_hwlat_sample()
143 	entry->seqnum			= sample->seqnum;  in trace_hwlat_sample()
144 	entry->duration			= sample->duration;  in trace_hwlat_sample()
145 	entry->outer_duration		= sample->outer_duration;  in trace_hwlat_sample()
146 	entry->timestamp		= sample->timestamp;  in trace_hwlat_sample()
147 	entry->nmi_total_ts		= sample->nmi_total_ts;  in trace_hwlat_sample()
148 	entry->nmi_count		= sample->nmi_count;  in trace_hwlat_sample()
149 	entry->count			= sample->count;  in trace_hwlat_sample()
159 #define time_sub(a, b)	((a) - (b))
167 	if (!kdata->kthread)  in trace_hwlat_callback()
176 			kdata->nmi_ts_start = time_get();  in trace_hwlat_callback()
178 			kdata->nmi_total_ts += time_get() - kdata->nmi_ts_start;  in trace_hwlat_callback()
182 		kdata->nmi_count++;  in trace_hwlat_callback()
186  * hwlat_err - report a hwlat error.
191 	trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, msg);	\
195  * get_sample - sample the CPU TSC and look for likely hardware latencies
198  * hardware-induced latency. Called with interrupts disabled and with
211 	int ret = -1;  in get_sample()
216 	kdata->nmi_total_ts = 0;  in get_sample()
217 	kdata->nmi_count = 0;  in get_sample()
287 		if (kdata->nmi_total_ts)  in get_sample()
288 			do_div(kdata->nmi_total_ts, NSEC_PER_USEC);  in get_sample()
294 		s.nmi_total_ts = kdata->nmi_total_ts;  in get_sample()
295 		s.nmi_count = kdata->nmi_count;  in get_sample()
302 		if (latency > tr->max_latency) {  in get_sample()
303 			tr->max_latency = latency;  in get_sample()
322 	 * of this thread, then stop migrating for the duration  in move_to_next_cpu()
325 	if (!cpumask_equal(current_mask, current->cpus_ptr))  in move_to_next_cpu()
329 	cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask);  in move_to_next_cpu()
347 	pr_info(BANNER "cpumask changed while in round-robin mode, switching to mode none\n");  in move_to_next_cpu()
351  * kthread_fn - The CPU time sampling/hardware latency detection kernel thread
374 		interval = hwlat_data.sample_window - hwlat_data.sample_width;  in kthread_fn()
391  * stop_stop_kthread - Inform the hardware latency sampling/detector kthread to stop
394  * tells it to stop sampling now. Use this on unload and at system shutdown.
402 	kthread = kdata->kthread;  in stop_single_kthread()
408 	kdata->kthread = NULL;  in stop_single_kthread()
416  * start_single_kthread - Kick off the hardware latency sampling/detector kthread
429 	if (kdata->kthread)  in start_single_kthread()
436 		return -ENOMEM;  in start_single_kthread()
440 	cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask);  in start_single_kthread()
451 	kdata->kthread = kthread;  in start_single_kthread()
460  * stop_cpu_kthread - Stop a hwlat cpu kthread
473  * stop_per_cpu_kthreads - Inform the hardware latency sampling/detector kthread to stop
476  * tells it to stop sampling now. Use this on unload and at system shutdown.
489  * start_cpu_kthread - Start a hwlat cpu kthread
502 		return -ENOMEM;  in start_cpu_kthread()
525 	if (!cpumask_test_cpu(cpu, tr->tracing_cpumask))  in hwlat_hotplug_workfn()
539  * hwlat_cpu_init - CPU hotplug online callback function
548  * hwlat_cpu_die - CPU hotplug offline callback function
575  * start_per_cpu_kthreads - Kick off the hardware latency sampling/detector kthreads
591 	cpumask_and(current_mask, cpu_online_mask, tr->tracing_cpumask);  in start_per_cpu_kthreads()
610 	int mode = *pos;  in s_mode_start()  local
614 	if (mode >= MODE_MAX)  in s_mode_start()
622 	int mode = ++(*pos);  in s_mode_next()  local
624 	if (mode >= MODE_MAX)  in s_mode_next()
633 	int mode = *pos;  in s_mode_show()  local
635 	if (mode == hwlat_data.thread_mode)  in s_mode_show()
636 		seq_printf(s, "[%s]", thread_mode_str[mode]);  in s_mode_show()
638 		seq_printf(s, "%s", thread_mode_str[mode]);  in s_mode_show()
640 	if (mode < MODE_MAX - 1) /* if mode is any but last */  in s_mode_show()
656 	.stop		= s_mode_stop
668  * hwlat_mode_write - Write function for "mode" entry
674  * This function provides a write implementation for the "mode" interface
677  * startup and lets the scheduler handle the migration. The default mode is
678  * the "round-robin" one, in which a single hwlatd thread runs, migrating
679  * among the allowed CPUs in a round-robin fashion. The "per-cpu" mode
686 	const char *mode;  in hwlat_mode_write()  local
691 		return -EINVAL;  in hwlat_mode_write()
694 		return -EFAULT;  in hwlat_mode_write()
698 	mode = strstrip(buf);  in hwlat_mode_write()
700 	ret = -EINVAL;  in hwlat_mode_write()
703 	 * trace_types_lock is taken to avoid concurrency on start/stop  in hwlat_mode_write()
713 		if (strcmp(mode, thread_mode_str[i]) == 0) {  in hwlat_mode_write()
764  * init_tracefs - A function to initialize the tracefs interface files
778 		return -ENOMEM;  in init_tracefs()
782 		return -ENOMEM;  in init_tracefs()
798 	hwlat_thread_mode = trace_create_file("mode", TRACE_MODE_WRITE,  in init_tracefs()
809 	return -ENOMEM;  in init_tracefs()
836 		return -EBUSY;  in hwlat_tracer_init()
841 	tr->max_latency = 0;  in hwlat_tracer_init()
873 	.stop		= hwlat_tracer_stop,