1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  (C) 2010,2011       Thomas Renninger <trenn@suse.de>, Novell Inc.
4  */
5 
6 #if defined(__i386__) || defined(__x86_64__)
7 
8 #include <stdio.h>
9 #include <stdint.h>
10 #include <stdlib.h>
11 #include <string.h>
12 #include <limits.h>
13 
14 #include <cpufreq.h>
15 
16 #include "helpers/helpers.h"
17 #include "idle_monitor/cpupower-monitor.h"
18 
19 #define MSR_APERF	0xE8
20 #define MSR_MPERF	0xE7
21 
22 #define RDPRU ".byte 0x0f, 0x01, 0xfd"
23 #define RDPRU_ECX_MPERF	0
24 #define RDPRU_ECX_APERF	1
25 
26 #define MSR_TSC	0x10
27 
28 #define MSR_AMD_HWCR 0xc0010015
29 
30 enum mperf_id { C0 = 0, Cx, AVG_FREQ, MPERF_CSTATE_COUNT };
31 
32 static int mperf_get_count_percent(unsigned int self_id, double *percent,
33 				   unsigned int cpu);
34 static int mperf_get_count_freq(unsigned int id, unsigned long long *count,
35 				unsigned int cpu);
36 static struct timespec time_start, time_end;
37 
38 static cstate_t mperf_cstates[MPERF_CSTATE_COUNT] = {
39 	{
40 		.name			= "C0",
41 		.desc			= N_("Processor Core not idle"),
42 		.id			= C0,
43 		.range			= RANGE_THREAD,
44 		.get_count_percent	= mperf_get_count_percent,
45 	},
46 	{
47 		.name			= "Cx",
48 		.desc			= N_("Processor Core in an idle state"),
49 		.id			= Cx,
50 		.range			= RANGE_THREAD,
51 		.get_count_percent	= mperf_get_count_percent,
52 	},
53 
54 	{
55 		.name			= "Freq",
56 		.desc			= N_("Average Frequency (including boost) in MHz"),
57 		.id			= AVG_FREQ,
58 		.range			= RANGE_THREAD,
59 		.get_count		= mperf_get_count_freq,
60 	},
61 };
62 
63 enum MAX_FREQ_MODE { MAX_FREQ_SYSFS, MAX_FREQ_TSC_REF };
64 static int max_freq_mode;
65 /*
66  * The max frequency mperf is ticking at (in C0), either retrieved via:
67  *   1) calculated after measurements if we know TSC ticks at mperf/P0 frequency
68  *   2) cpufreq /sys/devices/.../cpu0/cpufreq/cpuinfo_max_freq at init time
69  * 1. Is preferred as it also works without cpufreq subsystem (e.g. on Xen)
70  */
71 static unsigned long max_frequency;
72 
73 static unsigned long long *tsc_at_measure_start;
74 static unsigned long long *tsc_at_measure_end;
75 static unsigned long long *mperf_previous_count;
76 static unsigned long long *aperf_previous_count;
77 static unsigned long long *mperf_current_count;
78 static unsigned long long *aperf_current_count;
79 
80 /* valid flag for all CPUs. If a MSR read failed it will be zero */
81 static int *is_valid;
82 
mperf_get_tsc(unsigned long long * tsc)83 static int mperf_get_tsc(unsigned long long *tsc)
84 {
85 	int ret;
86 
87 	ret = read_msr(base_cpu, MSR_TSC, tsc);
88 	if (ret)
89 		dprint("Reading TSC MSR failed, returning %llu\n", *tsc);
90 	return ret;
91 }
92 
get_aperf_mperf(int cpu,unsigned long long * aval,unsigned long long * mval)93 static int get_aperf_mperf(int cpu, unsigned long long *aval,
94 				    unsigned long long *mval)
95 {
96 	unsigned long low_a, high_a;
97 	unsigned long low_m, high_m;
98 	int ret;
99 
100 	/*
101 	 * Running on the cpu from which we read the registers will
102 	 * prevent APERF/MPERF from going out of sync because of IPI
103 	 * latency introduced by read_msr()s.
104 	 */
105 	if (mperf_monitor.flags.per_cpu_schedule) {
106 		if (bind_cpu(cpu))
107 			return 1;
108 	}
109 
110 	if (cpupower_cpu_info.caps & CPUPOWER_CAP_AMD_RDPRU) {
111 		asm volatile(RDPRU
112 			     : "=a" (low_a), "=d" (high_a)
113 			     : "c" (RDPRU_ECX_APERF));
114 		asm volatile(RDPRU
115 			     : "=a" (low_m), "=d" (high_m)
116 			     : "c" (RDPRU_ECX_MPERF));
117 
118 		*aval = ((low_a) | (high_a) << 32);
119 		*mval = ((low_m) | (high_m) << 32);
120 
121 		return 0;
122 	}
123 
124 	ret  = read_msr(cpu, MSR_APERF, aval);
125 	ret |= read_msr(cpu, MSR_MPERF, mval);
126 
127 	return ret;
128 }
129 
mperf_init_stats(unsigned int cpu)130 static int mperf_init_stats(unsigned int cpu)
131 {
132 	unsigned long long aval, mval;
133 	int ret;
134 
135 	ret = get_aperf_mperf(cpu, &aval, &mval);
136 	aperf_previous_count[cpu] = aval;
137 	mperf_previous_count[cpu] = mval;
138 	is_valid[cpu] = !ret;
139 
140 	return 0;
141 }
142 
mperf_measure_stats(unsigned int cpu)143 static int mperf_measure_stats(unsigned int cpu)
144 {
145 	unsigned long long aval, mval;
146 	int ret;
147 
148 	ret = get_aperf_mperf(cpu, &aval, &mval);
149 	aperf_current_count[cpu] = aval;
150 	mperf_current_count[cpu] = mval;
151 	is_valid[cpu] = !ret;
152 
153 	return 0;
154 }
155 
mperf_get_count_percent(unsigned int id,double * percent,unsigned int cpu)156 static int mperf_get_count_percent(unsigned int id, double *percent,
157 				   unsigned int cpu)
158 {
159 	unsigned long long aperf_diff, mperf_diff, tsc_diff;
160 	unsigned long long timediff;
161 
162 	if (!is_valid[cpu])
163 		return -1;
164 
165 	if (id != C0 && id != Cx)
166 		return -1;
167 
168 	mperf_diff = mperf_current_count[cpu] - mperf_previous_count[cpu];
169 	aperf_diff = aperf_current_count[cpu] - aperf_previous_count[cpu];
170 
171 	if (max_freq_mode == MAX_FREQ_TSC_REF) {
172 		tsc_diff = tsc_at_measure_end[cpu] - tsc_at_measure_start[cpu];
173 		*percent = 100.0 * mperf_diff / tsc_diff;
174 		dprint("%s: TSC Ref - mperf_diff: %llu, tsc_diff: %llu\n",
175 		       mperf_cstates[id].name, mperf_diff, tsc_diff);
176 	} else if (max_freq_mode == MAX_FREQ_SYSFS) {
177 		timediff = max_frequency * timespec_diff_us(time_start, time_end);
178 		*percent = 100.0 * mperf_diff / timediff;
179 		dprint("%s: MAXFREQ - mperf_diff: %llu, time_diff: %llu\n",
180 		       mperf_cstates[id].name, mperf_diff, timediff);
181 	} else
182 		return -1;
183 
184 	if (id == Cx)
185 		*percent = 100.0 - *percent;
186 
187 	dprint("%s: previous: %llu - current: %llu - (%u)\n",
188 		mperf_cstates[id].name, mperf_diff, aperf_diff, cpu);
189 	dprint("%s: %f\n", mperf_cstates[id].name, *percent);
190 	return 0;
191 }
192 
mperf_get_count_freq(unsigned int id,unsigned long long * count,unsigned int cpu)193 static int mperf_get_count_freq(unsigned int id, unsigned long long *count,
194 				unsigned int cpu)
195 {
196 	unsigned long long aperf_diff, mperf_diff, time_diff, tsc_diff;
197 
198 	if (id != AVG_FREQ)
199 		return 1;
200 
201 	if (!is_valid[cpu])
202 		return -1;
203 
204 	mperf_diff = mperf_current_count[cpu] - mperf_previous_count[cpu];
205 	aperf_diff = aperf_current_count[cpu] - aperf_previous_count[cpu];
206 
207 	if (max_freq_mode == MAX_FREQ_TSC_REF) {
208 		/* Calculate max_freq from TSC count */
209 		tsc_diff = tsc_at_measure_end[cpu] - tsc_at_measure_start[cpu];
210 		time_diff = timespec_diff_us(time_start, time_end);
211 		max_frequency = tsc_diff / time_diff;
212 	}
213 
214 	*count = max_frequency * ((double)aperf_diff / mperf_diff);
215 	dprint("%s: Average freq based on %s maximum frequency:\n",
216 	       mperf_cstates[id].name,
217 	       (max_freq_mode == MAX_FREQ_TSC_REF) ? "TSC calculated" : "sysfs read");
218 	dprint("max_frequency: %lu\n", max_frequency);
219 	dprint("aperf_diff: %llu\n", aperf_diff);
220 	dprint("mperf_diff: %llu\n", mperf_diff);
221 	dprint("avg freq:   %llu\n", *count);
222 	return 0;
223 }
224 
mperf_start(void)225 static int mperf_start(void)
226 {
227 	int cpu;
228 
229 	clock_gettime(CLOCK_REALTIME, &time_start);
230 
231 	for (cpu = 0; cpu < cpu_count; cpu++) {
232 		mperf_get_tsc(&tsc_at_measure_start[cpu]);
233 		mperf_init_stats(cpu);
234 	}
235 
236 	return 0;
237 }
238 
mperf_stop(void)239 static int mperf_stop(void)
240 {
241 	int cpu;
242 
243 	for (cpu = 0; cpu < cpu_count; cpu++) {
244 		mperf_measure_stats(cpu);
245 		mperf_get_tsc(&tsc_at_measure_end[cpu]);
246 	}
247 
248 	clock_gettime(CLOCK_REALTIME, &time_end);
249 	return 0;
250 }
251 
252 /*
253  * Mperf register is defined to tick at P0 (maximum) frequency
254  *
255  * Instead of reading out P0 which can be tricky to read out from HW,
256  * we use TSC counter if it reliably ticks at P0/mperf frequency.
257  *
258  * Still try to fall back to:
259  * /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq
260  * on older Intel HW without invariant TSC feature.
261  * Or on AMD machines where TSC does not tick at P0 (do not exist yet, but
262  * it's still double checked (MSR_AMD_HWCR)).
263  *
264  * On these machines the user would still get useful mperf
265  * stats when acpi-cpufreq driver is loaded.
266  */
init_maxfreq_mode(void)267 static int init_maxfreq_mode(void)
268 {
269 	int ret;
270 	unsigned long long hwcr;
271 	unsigned long min;
272 
273 	if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_INV_TSC))
274 		goto use_sysfs;
275 
276 	if (cpupower_cpu_info.vendor == X86_VENDOR_AMD ||
277 	    cpupower_cpu_info.vendor == X86_VENDOR_HYGON) {
278 		/* MSR_AMD_HWCR tells us whether TSC runs at P0/mperf
279 		 * freq.
280 		 * A test whether hwcr is accessable/available would be:
281 		 * (cpupower_cpu_info.family > 0x10 ||
282 		 *   cpupower_cpu_info.family == 0x10 &&
283 		 *   cpupower_cpu_info.model >= 0x2))
284 		 * This should be the case for all aperf/mperf
285 		 * capable AMD machines and is therefore safe to test here.
286 		 * Compare with Linus kernel git commit: acf01734b1747b1ec4
287 		 */
288 		ret = read_msr(0, MSR_AMD_HWCR, &hwcr);
289 		/*
290 		 * If the MSR read failed, assume a Xen system that did
291 		 * not explicitly provide access to it and assume TSC works
292 		*/
293 		if (ret != 0) {
294 			dprint("TSC read 0x%x failed - assume TSC working\n",
295 			       MSR_AMD_HWCR);
296 			return 0;
297 		} else if (1 & (hwcr >> 24)) {
298 			max_freq_mode = MAX_FREQ_TSC_REF;
299 			return 0;
300 		} else { /* Use sysfs max frequency if available */ }
301 	} else if (cpupower_cpu_info.vendor == X86_VENDOR_INTEL) {
302 		/*
303 		 * On Intel we assume mperf (in C0) is ticking at same
304 		 * rate than TSC
305 		 */
306 		max_freq_mode = MAX_FREQ_TSC_REF;
307 		return 0;
308 	}
309 use_sysfs:
310 	if (cpufreq_get_hardware_limits(0, &min, &max_frequency)) {
311 		dprint("Cannot retrieve max freq from cpufreq kernel "
312 		       "subsystem\n");
313 		return -1;
314 	}
315 	max_freq_mode = MAX_FREQ_SYSFS;
316 	max_frequency /= 1000; /* Default automatically to MHz value */
317 	return 0;
318 }
319 
320 /*
321  * This monitor provides:
322  *
323  * 1) Average frequency a CPU resided in
324  *    This always works if the CPU has aperf/mperf capabilities
325  *
326  * 2) C0 and Cx (any sleep state) time a CPU resided in
327  *    Works if mperf timer stops ticking in sleep states which
328  *    seem to be the case on all current HW.
329  * Both is directly retrieved from HW registers and is independent
330  * from kernel statistics.
331  */
332 struct cpuidle_monitor mperf_monitor;
mperf_register(void)333 struct cpuidle_monitor *mperf_register(void)
334 {
335 	if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_APERF))
336 		return NULL;
337 
338 	if (init_maxfreq_mode())
339 		return NULL;
340 
341 	if (cpupower_cpu_info.vendor == X86_VENDOR_AMD)
342 		mperf_monitor.flags.per_cpu_schedule = 1;
343 
344 	/* Free this at program termination */
345 	is_valid = calloc(cpu_count, sizeof(int));
346 	mperf_previous_count = calloc(cpu_count, sizeof(unsigned long long));
347 	aperf_previous_count = calloc(cpu_count, sizeof(unsigned long long));
348 	mperf_current_count = calloc(cpu_count, sizeof(unsigned long long));
349 	aperf_current_count = calloc(cpu_count, sizeof(unsigned long long));
350 	tsc_at_measure_start = calloc(cpu_count, sizeof(unsigned long long));
351 	tsc_at_measure_end = calloc(cpu_count, sizeof(unsigned long long));
352 	mperf_monitor.name_len = strlen(mperf_monitor.name);
353 	return &mperf_monitor;
354 }
355 
mperf_unregister(void)356 void mperf_unregister(void)
357 {
358 	free(mperf_previous_count);
359 	free(aperf_previous_count);
360 	free(mperf_current_count);
361 	free(aperf_current_count);
362 	free(tsc_at_measure_start);
363 	free(tsc_at_measure_end);
364 	free(is_valid);
365 }
366 
367 struct cpuidle_monitor mperf_monitor = {
368 	.name			= "Mperf",
369 	.hw_states_num		= MPERF_CSTATE_COUNT,
370 	.hw_states		= mperf_cstates,
371 	.start			= mperf_start,
372 	.stop			= mperf_stop,
373 	.do_register		= mperf_register,
374 	.unregister		= mperf_unregister,
375 	.flags.needs_root	= 1,
376 	.overflow_s		= 922000000 /* 922337203 seconds TSC overflow
377 					       at 20GHz */
378 };
379 #endif /* #if defined(__i386__) || defined(__x86_64__) */
380