1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  drivers/cpufreq/cpufreq_ondemand.c
4  *
5  *  Copyright (C)  2001 Russell King
6  *            (C)  2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
7  *                      Jun Nakajima <jun.nakajima@intel.com>
8  */
9 
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11 
12 #include <linux/cpu.h>
13 #include <linux/percpu-defs.h>
14 #include <linux/slab.h>
15 #include <linux/tick.h>
16 #include <linux/sched/cpufreq.h>
17 
18 #include "cpufreq_ondemand.h"
19 
20 /* On-demand governor macros */
21 #define DEF_FREQUENCY_UP_THRESHOLD		(80)
22 #define DEF_SAMPLING_DOWN_FACTOR		(1)
23 #define MAX_SAMPLING_DOWN_FACTOR		(100000)
24 #define MICRO_FREQUENCY_UP_THRESHOLD		(95)
25 #define MICRO_FREQUENCY_MIN_SAMPLE_RATE		(10000)
26 #define MIN_FREQUENCY_UP_THRESHOLD		(1)
27 #define MAX_FREQUENCY_UP_THRESHOLD		(100)
28 
29 static struct od_ops od_ops;
30 
31 static unsigned int default_powersave_bias;
32 
33 /*
34  * Not all CPUs want IO time to be accounted as busy; this depends on how
35  * efficient idling at a higher frequency/voltage is.
36  * Pavel Machek says this is not so for various generations of AMD and old
37  * Intel systems.
38  * Mike Chan (android.com) claims this is also not true for ARM.
39  * Because of this, whitelist specific known (series) of CPUs by default, and
40  * leave all others up to the user.
41  */
42 static int should_io_be_busy(void)
43 {
44 #if defined(CONFIG_X86)
45 	/*
46 	 * For Intel, Core 2 (model 15) and later have an efficient idle.
47 	 */
48 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
49 			boot_cpu_data.x86 == 6 &&
50 			boot_cpu_data.x86_model >= 15)
51 		return 1;
52 #endif
53 	return 0;
54 }
55 
56 /*
57  * Find right freq to be set now with powersave_bias on.
58  * Returns the freq_hi to be used right now and will set freq_hi_delay_us,
59  * freq_lo, and freq_lo_delay_us in percpu area for averaging freqs.
60  */
61 static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy,
62 		unsigned int freq_next, unsigned int relation)
63 {
64 	unsigned int freq_req, freq_reduc, freq_avg;
65 	unsigned int freq_hi, freq_lo;
66 	unsigned int index;
67 	unsigned int delay_hi_us;
68 	struct policy_dbs_info *policy_dbs = policy->governor_data;
69 	struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
70 	struct dbs_data *dbs_data = policy_dbs->dbs_data;
71 	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
72 	struct cpufreq_frequency_table *freq_table = policy->freq_table;
73 
74 	if (!freq_table) {
75 		dbs_info->freq_lo = 0;
76 		dbs_info->freq_lo_delay_us = 0;
77 		return freq_next;
78 	}
79 
80 	index = cpufreq_frequency_table_target(policy, freq_next, relation);
81 	freq_req = freq_table[index].frequency;
82 	freq_reduc = freq_req * od_tuners->powersave_bias / 1000;
83 	freq_avg = freq_req - freq_reduc;
84 
85 	/* Find freq bounds for freq_avg in freq_table */
86 	index = cpufreq_table_find_index_h(policy, freq_avg);
87 	freq_lo = freq_table[index].frequency;
88 	index = cpufreq_table_find_index_l(policy, freq_avg);
89 	freq_hi = freq_table[index].frequency;
90 
91 	/* Find out how long we have to be in hi and lo freqs */
92 	if (freq_hi == freq_lo) {
93 		dbs_info->freq_lo = 0;
94 		dbs_info->freq_lo_delay_us = 0;
95 		return freq_lo;
96 	}
97 	delay_hi_us = (freq_avg - freq_lo) * dbs_data->sampling_rate;
98 	delay_hi_us += (freq_hi - freq_lo) / 2;
99 	delay_hi_us /= freq_hi - freq_lo;
100 	dbs_info->freq_hi_delay_us = delay_hi_us;
101 	dbs_info->freq_lo = freq_lo;
102 	dbs_info->freq_lo_delay_us = dbs_data->sampling_rate - delay_hi_us;
103 	return freq_hi;
104 }
105 
106 static void ondemand_powersave_bias_init(struct cpufreq_policy *policy)
107 {
108 	struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data);
109 
110 	dbs_info->freq_lo = 0;
111 }
112 
113 static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq)
114 {
115 	struct policy_dbs_info *policy_dbs = policy->governor_data;
116 	struct dbs_data *dbs_data = policy_dbs->dbs_data;
117 	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
118 
119 	if (od_tuners->powersave_bias)
120 		freq = od_ops.powersave_bias_target(policy, freq,
121 				CPUFREQ_RELATION_H);
122 	else if (policy->cur == policy->max)
123 		return;
124 
125 	__cpufreq_driver_target(policy, freq, od_tuners->powersave_bias ?
126 			CPUFREQ_RELATION_L : CPUFREQ_RELATION_H);
127 }
128 
129 /*
130  * Every sampling_rate, we check, if current idle time is less than 20%
131  * (default), then we try to increase frequency. Else, we adjust the frequency
132  * proportional to load.
133  */
134 static void od_update(struct cpufreq_policy *policy)
135 {
136 	struct policy_dbs_info *policy_dbs = policy->governor_data;
137 	struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
138 	struct dbs_data *dbs_data = policy_dbs->dbs_data;
139 	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
140 	unsigned int load = dbs_update(policy);
141 
142 	dbs_info->freq_lo = 0;
143 
144 	/* Check for frequency increase */
145 	if (load > dbs_data->up_threshold) {
146 		/* If switching to max speed, apply sampling_down_factor */
147 		if (policy->cur < policy->max)
148 			policy_dbs->rate_mult = dbs_data->sampling_down_factor;
149 		dbs_freq_increase(policy, policy->max);
150 	} else {
151 		/* Calculate the next frequency proportional to load */
152 		unsigned int freq_next, min_f, max_f;
153 
154 		min_f = policy->cpuinfo.min_freq;
155 		max_f = policy->cpuinfo.max_freq;
156 		freq_next = min_f + load * (max_f - min_f) / 100;
157 
158 		/* No longer fully busy, reset rate_mult */
159 		policy_dbs->rate_mult = 1;
160 
161 		if (od_tuners->powersave_bias)
162 			freq_next = od_ops.powersave_bias_target(policy,
163 								 freq_next,
164 								 CPUFREQ_RELATION_L);
165 
166 		__cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_C);
167 	}
168 }
169 
170 static unsigned int od_dbs_update(struct cpufreq_policy *policy)
171 {
172 	struct policy_dbs_info *policy_dbs = policy->governor_data;
173 	struct dbs_data *dbs_data = policy_dbs->dbs_data;
174 	struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
175 	int sample_type = dbs_info->sample_type;
176 
177 	/* Common NORMAL_SAMPLE setup */
178 	dbs_info->sample_type = OD_NORMAL_SAMPLE;
179 	/*
180 	 * OD_SUB_SAMPLE doesn't make sense if sample_delay_ns is 0, so ignore
181 	 * it then.
182 	 */
183 	if (sample_type == OD_SUB_SAMPLE && policy_dbs->sample_delay_ns > 0) {
184 		__cpufreq_driver_target(policy, dbs_info->freq_lo,
185 					CPUFREQ_RELATION_H);
186 		return dbs_info->freq_lo_delay_us;
187 	}
188 
189 	od_update(policy);
190 
191 	if (dbs_info->freq_lo) {
192 		/* Setup SUB_SAMPLE */
193 		dbs_info->sample_type = OD_SUB_SAMPLE;
194 		return dbs_info->freq_hi_delay_us;
195 	}
196 
197 	return dbs_data->sampling_rate * policy_dbs->rate_mult;
198 }
199 
200 /************************** sysfs interface ************************/
201 static struct dbs_governor od_dbs_gov;
202 
203 static ssize_t store_io_is_busy(struct gov_attr_set *attr_set, const char *buf,
204 				size_t count)
205 {
206 	struct dbs_data *dbs_data = to_dbs_data(attr_set);
207 	unsigned int input;
208 	int ret;
209 
210 	ret = sscanf(buf, "%u", &input);
211 	if (ret != 1)
212 		return -EINVAL;
213 	dbs_data->io_is_busy = !!input;
214 
215 	/* we need to re-evaluate prev_cpu_idle */
216 	gov_update_cpu_data(dbs_data);
217 
218 	return count;
219 }
220 
221 static ssize_t store_up_threshold(struct gov_attr_set *attr_set,
222 				  const char *buf, size_t count)
223 {
224 	struct dbs_data *dbs_data = to_dbs_data(attr_set);
225 	unsigned int input;
226 	int ret;
227 	ret = sscanf(buf, "%u", &input);
228 
229 	if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
230 			input < MIN_FREQUENCY_UP_THRESHOLD) {
231 		return -EINVAL;
232 	}
233 
234 	dbs_data->up_threshold = input;
235 	return count;
236 }
237 
238 static ssize_t store_sampling_down_factor(struct gov_attr_set *attr_set,
239 					  const char *buf, size_t count)
240 {
241 	struct dbs_data *dbs_data = to_dbs_data(attr_set);
242 	struct policy_dbs_info *policy_dbs;
243 	unsigned int input;
244 	int ret;
245 	ret = sscanf(buf, "%u", &input);
246 
247 	if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
248 		return -EINVAL;
249 
250 	dbs_data->sampling_down_factor = input;
251 
252 	/* Reset down sampling multiplier in case it was active */
253 	list_for_each_entry(policy_dbs, &attr_set->policy_list, list) {
254 		/*
255 		 * Doing this without locking might lead to using different
256 		 * rate_mult values in od_update() and od_dbs_update().
257 		 */
258 		mutex_lock(&policy_dbs->update_mutex);
259 		policy_dbs->rate_mult = 1;
260 		mutex_unlock(&policy_dbs->update_mutex);
261 	}
262 
263 	return count;
264 }
265 
266 static ssize_t store_ignore_nice_load(struct gov_attr_set *attr_set,
267 				      const char *buf, size_t count)
268 {
269 	struct dbs_data *dbs_data = to_dbs_data(attr_set);
270 	unsigned int input;
271 	int ret;
272 
273 	ret = sscanf(buf, "%u", &input);
274 	if (ret != 1)
275 		return -EINVAL;
276 
277 	if (input > 1)
278 		input = 1;
279 
280 	if (input == dbs_data->ignore_nice_load) { /* nothing to do */
281 		return count;
282 	}
283 	dbs_data->ignore_nice_load = input;
284 
285 	/* we need to re-evaluate prev_cpu_idle */
286 	gov_update_cpu_data(dbs_data);
287 
288 	return count;
289 }
290 
291 static ssize_t store_powersave_bias(struct gov_attr_set *attr_set,
292 				    const char *buf, size_t count)
293 {
294 	struct dbs_data *dbs_data = to_dbs_data(attr_set);
295 	struct od_dbs_tuners *od_tuners = dbs_data->tuners;
296 	struct policy_dbs_info *policy_dbs;
297 	unsigned int input;
298 	int ret;
299 	ret = sscanf(buf, "%u", &input);
300 
301 	if (ret != 1)
302 		return -EINVAL;
303 
304 	if (input > 1000)
305 		input = 1000;
306 
307 	od_tuners->powersave_bias = input;
308 
309 	list_for_each_entry(policy_dbs, &attr_set->policy_list, list)
310 		ondemand_powersave_bias_init(policy_dbs->policy);
311 
312 	return count;
313 }
314 
315 gov_show_one_common(sampling_rate);
316 gov_show_one_common(up_threshold);
317 gov_show_one_common(sampling_down_factor);
318 gov_show_one_common(ignore_nice_load);
319 gov_show_one_common(io_is_busy);
320 gov_show_one(od, powersave_bias);
321 
322 gov_attr_rw(sampling_rate);
323 gov_attr_rw(io_is_busy);
324 gov_attr_rw(up_threshold);
325 gov_attr_rw(sampling_down_factor);
326 gov_attr_rw(ignore_nice_load);
327 gov_attr_rw(powersave_bias);
328 
329 static struct attribute *od_attributes[] = {
330 	&sampling_rate.attr,
331 	&up_threshold.attr,
332 	&sampling_down_factor.attr,
333 	&ignore_nice_load.attr,
334 	&powersave_bias.attr,
335 	&io_is_busy.attr,
336 	NULL
337 };
338 
339 /************************** sysfs end ************************/
340 
341 static struct policy_dbs_info *od_alloc(void)
342 {
343 	struct od_policy_dbs_info *dbs_info;
344 
345 	dbs_info = kzalloc(sizeof(*dbs_info), GFP_KERNEL);
346 	return dbs_info ? &dbs_info->policy_dbs : NULL;
347 }
348 
349 static void od_free(struct policy_dbs_info *policy_dbs)
350 {
351 	kfree(to_dbs_info(policy_dbs));
352 }
353 
354 static int od_init(struct dbs_data *dbs_data)
355 {
356 	struct od_dbs_tuners *tuners;
357 	u64 idle_time;
358 	int cpu;
359 
360 	tuners = kzalloc(sizeof(*tuners), GFP_KERNEL);
361 	if (!tuners)
362 		return -ENOMEM;
363 
364 	cpu = get_cpu();
365 	idle_time = get_cpu_idle_time_us(cpu, NULL);
366 	put_cpu();
367 	if (idle_time != -1ULL) {
368 		/* Idle micro accounting is supported. Use finer thresholds */
369 		dbs_data->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
370 	} else {
371 		dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
372 	}
373 
374 	dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
375 	dbs_data->ignore_nice_load = 0;
376 	tuners->powersave_bias = default_powersave_bias;
377 	dbs_data->io_is_busy = should_io_be_busy();
378 
379 	dbs_data->tuners = tuners;
380 	return 0;
381 }
382 
383 static void od_exit(struct dbs_data *dbs_data)
384 {
385 	kfree(dbs_data->tuners);
386 }
387 
388 static void od_start(struct cpufreq_policy *policy)
389 {
390 	struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data);
391 
392 	dbs_info->sample_type = OD_NORMAL_SAMPLE;
393 	ondemand_powersave_bias_init(policy);
394 }
395 
396 static struct od_ops od_ops = {
397 	.powersave_bias_target = generic_powersave_bias_target,
398 };
399 
400 static struct dbs_governor od_dbs_gov = {
401 	.gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("ondemand"),
402 	.kobj_type = { .default_attrs = od_attributes },
403 	.gov_dbs_update = od_dbs_update,
404 	.alloc = od_alloc,
405 	.free = od_free,
406 	.init = od_init,
407 	.exit = od_exit,
408 	.start = od_start,
409 };
410 
411 #define CPU_FREQ_GOV_ONDEMAND	(&od_dbs_gov.gov)
412 
413 static void od_set_powersave_bias(unsigned int powersave_bias)
414 {
415 	unsigned int cpu;
416 	cpumask_t done;
417 
418 	default_powersave_bias = powersave_bias;
419 	cpumask_clear(&done);
420 
421 	get_online_cpus();
422 	for_each_online_cpu(cpu) {
423 		struct cpufreq_policy *policy;
424 		struct policy_dbs_info *policy_dbs;
425 		struct dbs_data *dbs_data;
426 		struct od_dbs_tuners *od_tuners;
427 
428 		if (cpumask_test_cpu(cpu, &done))
429 			continue;
430 
431 		policy = cpufreq_cpu_get_raw(cpu);
432 		if (!policy || policy->governor != CPU_FREQ_GOV_ONDEMAND)
433 			continue;
434 
435 		policy_dbs = policy->governor_data;
436 		if (!policy_dbs)
437 			continue;
438 
439 		cpumask_or(&done, &done, policy->cpus);
440 
441 		dbs_data = policy_dbs->dbs_data;
442 		od_tuners = dbs_data->tuners;
443 		od_tuners->powersave_bias = default_powersave_bias;
444 	}
445 	put_online_cpus();
446 }
447 
448 void od_register_powersave_bias_handler(unsigned int (*f)
449 		(struct cpufreq_policy *, unsigned int, unsigned int),
450 		unsigned int powersave_bias)
451 {
452 	od_ops.powersave_bias_target = f;
453 	od_set_powersave_bias(powersave_bias);
454 }
455 EXPORT_SYMBOL_GPL(od_register_powersave_bias_handler);
456 
457 void od_unregister_powersave_bias_handler(void)
458 {
459 	od_ops.powersave_bias_target = generic_powersave_bias_target;
460 	od_set_powersave_bias(0);
461 }
462 EXPORT_SYMBOL_GPL(od_unregister_powersave_bias_handler);
463 
464 static int __init cpufreq_gov_dbs_init(void)
465 {
466 	return cpufreq_register_governor(CPU_FREQ_GOV_ONDEMAND);
467 }
468 
469 static void __exit cpufreq_gov_dbs_exit(void)
470 {
471 	cpufreq_unregister_governor(CPU_FREQ_GOV_ONDEMAND);
472 }
473 
474 MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
475 MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
476 MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for "
477 	"Low Latency Frequency Transition capable processors");
478 MODULE_LICENSE("GPL");
479 
480 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
481 struct cpufreq_governor *cpufreq_default_governor(void)
482 {
483 	return CPU_FREQ_GOV_ONDEMAND;
484 }
485 
486 core_initcall(cpufreq_gov_dbs_init);
487 #else
488 module_init(cpufreq_gov_dbs_init);
489 #endif
490 module_exit(cpufreq_gov_dbs_exit);
491