1 /* 2 * drivers/cpufreq/cpufreq_ondemand.c 3 * 4 * Copyright (C) 2001 Russell King 5 * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>. 6 * Jun Nakajima <jun.nakajima@intel.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 */ 12 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/cpu.h> 16 #include <linux/percpu-defs.h> 17 #include <linux/slab.h> 18 #include <linux/tick.h> 19 20 #include "cpufreq_ondemand.h" 21 22 /* On-demand governor macros */ 23 #define DEF_FREQUENCY_UP_THRESHOLD (80) 24 #define DEF_SAMPLING_DOWN_FACTOR (1) 25 #define MAX_SAMPLING_DOWN_FACTOR (100000) 26 #define MICRO_FREQUENCY_UP_THRESHOLD (95) 27 #define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000) 28 #define MIN_FREQUENCY_UP_THRESHOLD (11) 29 #define MAX_FREQUENCY_UP_THRESHOLD (100) 30 31 static struct od_ops od_ops; 32 33 static unsigned int default_powersave_bias; 34 35 /* 36 * Not all CPUs want IO time to be accounted as busy; this depends on how 37 * efficient idling at a higher frequency/voltage is. 38 * Pavel Machek says this is not so for various generations of AMD and old 39 * Intel systems. 40 * Mike Chan (android.com) claims this is also not true for ARM. 41 * Because of this, whitelist specific known (series) of CPUs by default, and 42 * leave all others up to the user. 43 */ 44 static int should_io_be_busy(void) 45 { 46 #if defined(CONFIG_X86) 47 /* 48 * For Intel, Core 2 (model 15) and later have an efficient idle. 49 */ 50 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && 51 boot_cpu_data.x86 == 6 && 52 boot_cpu_data.x86_model >= 15) 53 return 1; 54 #endif 55 return 0; 56 } 57 58 /* 59 * Find right freq to be set now with powersave_bias on. 60 * Returns the freq_hi to be used right now and will set freq_hi_delay_us, 61 * freq_lo, and freq_lo_delay_us in percpu area for averaging freqs. 62 */ 63 static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy, 64 unsigned int freq_next, unsigned int relation) 65 { 66 unsigned int freq_req, freq_reduc, freq_avg; 67 unsigned int freq_hi, freq_lo; 68 unsigned int index = 0; 69 unsigned int delay_hi_us; 70 struct policy_dbs_info *policy_dbs = policy->governor_data; 71 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); 72 struct dbs_data *dbs_data = policy_dbs->dbs_data; 73 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 74 75 if (!dbs_info->freq_table) { 76 dbs_info->freq_lo = 0; 77 dbs_info->freq_lo_delay_us = 0; 78 return freq_next; 79 } 80 81 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next, 82 relation, &index); 83 freq_req = dbs_info->freq_table[index].frequency; 84 freq_reduc = freq_req * od_tuners->powersave_bias / 1000; 85 freq_avg = freq_req - freq_reduc; 86 87 /* Find freq bounds for freq_avg in freq_table */ 88 index = 0; 89 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg, 90 CPUFREQ_RELATION_H, &index); 91 freq_lo = dbs_info->freq_table[index].frequency; 92 index = 0; 93 cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg, 94 CPUFREQ_RELATION_L, &index); 95 freq_hi = dbs_info->freq_table[index].frequency; 96 97 /* Find out how long we have to be in hi and lo freqs */ 98 if (freq_hi == freq_lo) { 99 dbs_info->freq_lo = 0; 100 dbs_info->freq_lo_delay_us = 0; 101 return freq_lo; 102 } 103 delay_hi_us = (freq_avg - freq_lo) * dbs_data->sampling_rate; 104 delay_hi_us += (freq_hi - freq_lo) / 2; 105 delay_hi_us /= freq_hi - freq_lo; 106 dbs_info->freq_hi_delay_us = delay_hi_us; 107 dbs_info->freq_lo = freq_lo; 108 dbs_info->freq_lo_delay_us = dbs_data->sampling_rate - delay_hi_us; 109 return freq_hi; 110 } 111 112 static void ondemand_powersave_bias_init(struct cpufreq_policy *policy) 113 { 114 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); 115 116 dbs_info->freq_table = cpufreq_frequency_get_table(policy->cpu); 117 dbs_info->freq_lo = 0; 118 } 119 120 static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq) 121 { 122 struct policy_dbs_info *policy_dbs = policy->governor_data; 123 struct dbs_data *dbs_data = policy_dbs->dbs_data; 124 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 125 126 if (od_tuners->powersave_bias) 127 freq = od_ops.powersave_bias_target(policy, freq, 128 CPUFREQ_RELATION_H); 129 else if (policy->cur == policy->max) 130 return; 131 132 __cpufreq_driver_target(policy, freq, od_tuners->powersave_bias ? 133 CPUFREQ_RELATION_L : CPUFREQ_RELATION_H); 134 } 135 136 /* 137 * Every sampling_rate, we check, if current idle time is less than 20% 138 * (default), then we try to increase frequency. Else, we adjust the frequency 139 * proportional to load. 140 */ 141 static void od_update(struct cpufreq_policy *policy) 142 { 143 struct policy_dbs_info *policy_dbs = policy->governor_data; 144 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); 145 struct dbs_data *dbs_data = policy_dbs->dbs_data; 146 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 147 unsigned int load = dbs_update(policy); 148 149 dbs_info->freq_lo = 0; 150 151 /* Check for frequency increase */ 152 if (load > dbs_data->up_threshold) { 153 /* If switching to max speed, apply sampling_down_factor */ 154 if (policy->cur < policy->max) 155 policy_dbs->rate_mult = dbs_data->sampling_down_factor; 156 dbs_freq_increase(policy, policy->max); 157 } else { 158 /* Calculate the next frequency proportional to load */ 159 unsigned int freq_next, min_f, max_f; 160 161 min_f = policy->cpuinfo.min_freq; 162 max_f = policy->cpuinfo.max_freq; 163 freq_next = min_f + load * (max_f - min_f) / 100; 164 165 /* No longer fully busy, reset rate_mult */ 166 policy_dbs->rate_mult = 1; 167 168 if (od_tuners->powersave_bias) 169 freq_next = od_ops.powersave_bias_target(policy, 170 freq_next, 171 CPUFREQ_RELATION_L); 172 173 __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_C); 174 } 175 } 176 177 static unsigned int od_dbs_timer(struct cpufreq_policy *policy) 178 { 179 struct policy_dbs_info *policy_dbs = policy->governor_data; 180 struct dbs_data *dbs_data = policy_dbs->dbs_data; 181 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); 182 int sample_type = dbs_info->sample_type; 183 184 /* Common NORMAL_SAMPLE setup */ 185 dbs_info->sample_type = OD_NORMAL_SAMPLE; 186 /* 187 * OD_SUB_SAMPLE doesn't make sense if sample_delay_ns is 0, so ignore 188 * it then. 189 */ 190 if (sample_type == OD_SUB_SAMPLE && policy_dbs->sample_delay_ns > 0) { 191 __cpufreq_driver_target(policy, dbs_info->freq_lo, 192 CPUFREQ_RELATION_H); 193 return dbs_info->freq_lo_delay_us; 194 } 195 196 od_update(policy); 197 198 if (dbs_info->freq_lo) { 199 /* Setup timer for SUB_SAMPLE */ 200 dbs_info->sample_type = OD_SUB_SAMPLE; 201 return dbs_info->freq_hi_delay_us; 202 } 203 204 return dbs_data->sampling_rate * policy_dbs->rate_mult; 205 } 206 207 /************************** sysfs interface ************************/ 208 static struct dbs_governor od_dbs_gov; 209 210 static ssize_t store_io_is_busy(struct gov_attr_set *attr_set, const char *buf, 211 size_t count) 212 { 213 struct dbs_data *dbs_data = to_dbs_data(attr_set); 214 unsigned int input; 215 int ret; 216 217 ret = sscanf(buf, "%u", &input); 218 if (ret != 1) 219 return -EINVAL; 220 dbs_data->io_is_busy = !!input; 221 222 /* we need to re-evaluate prev_cpu_idle */ 223 gov_update_cpu_data(dbs_data); 224 225 return count; 226 } 227 228 static ssize_t store_up_threshold(struct gov_attr_set *attr_set, 229 const char *buf, size_t count) 230 { 231 struct dbs_data *dbs_data = to_dbs_data(attr_set); 232 unsigned int input; 233 int ret; 234 ret = sscanf(buf, "%u", &input); 235 236 if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD || 237 input < MIN_FREQUENCY_UP_THRESHOLD) { 238 return -EINVAL; 239 } 240 241 dbs_data->up_threshold = input; 242 return count; 243 } 244 245 static ssize_t store_sampling_down_factor(struct gov_attr_set *attr_set, 246 const char *buf, size_t count) 247 { 248 struct dbs_data *dbs_data = to_dbs_data(attr_set); 249 struct policy_dbs_info *policy_dbs; 250 unsigned int input; 251 int ret; 252 ret = sscanf(buf, "%u", &input); 253 254 if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1) 255 return -EINVAL; 256 257 dbs_data->sampling_down_factor = input; 258 259 /* Reset down sampling multiplier in case it was active */ 260 list_for_each_entry(policy_dbs, &attr_set->policy_list, list) { 261 /* 262 * Doing this without locking might lead to using different 263 * rate_mult values in od_update() and od_dbs_timer(). 264 */ 265 mutex_lock(&policy_dbs->timer_mutex); 266 policy_dbs->rate_mult = 1; 267 mutex_unlock(&policy_dbs->timer_mutex); 268 } 269 270 return count; 271 } 272 273 static ssize_t store_ignore_nice_load(struct gov_attr_set *attr_set, 274 const char *buf, size_t count) 275 { 276 struct dbs_data *dbs_data = to_dbs_data(attr_set); 277 unsigned int input; 278 int ret; 279 280 ret = sscanf(buf, "%u", &input); 281 if (ret != 1) 282 return -EINVAL; 283 284 if (input > 1) 285 input = 1; 286 287 if (input == dbs_data->ignore_nice_load) { /* nothing to do */ 288 return count; 289 } 290 dbs_data->ignore_nice_load = input; 291 292 /* we need to re-evaluate prev_cpu_idle */ 293 gov_update_cpu_data(dbs_data); 294 295 return count; 296 } 297 298 static ssize_t store_powersave_bias(struct gov_attr_set *attr_set, 299 const char *buf, size_t count) 300 { 301 struct dbs_data *dbs_data = to_dbs_data(attr_set); 302 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 303 struct policy_dbs_info *policy_dbs; 304 unsigned int input; 305 int ret; 306 ret = sscanf(buf, "%u", &input); 307 308 if (ret != 1) 309 return -EINVAL; 310 311 if (input > 1000) 312 input = 1000; 313 314 od_tuners->powersave_bias = input; 315 316 list_for_each_entry(policy_dbs, &attr_set->policy_list, list) 317 ondemand_powersave_bias_init(policy_dbs->policy); 318 319 return count; 320 } 321 322 gov_show_one_common(sampling_rate); 323 gov_show_one_common(up_threshold); 324 gov_show_one_common(sampling_down_factor); 325 gov_show_one_common(ignore_nice_load); 326 gov_show_one_common(min_sampling_rate); 327 gov_show_one_common(io_is_busy); 328 gov_show_one(od, powersave_bias); 329 330 gov_attr_rw(sampling_rate); 331 gov_attr_rw(io_is_busy); 332 gov_attr_rw(up_threshold); 333 gov_attr_rw(sampling_down_factor); 334 gov_attr_rw(ignore_nice_load); 335 gov_attr_rw(powersave_bias); 336 gov_attr_ro(min_sampling_rate); 337 338 static struct attribute *od_attributes[] = { 339 &min_sampling_rate.attr, 340 &sampling_rate.attr, 341 &up_threshold.attr, 342 &sampling_down_factor.attr, 343 &ignore_nice_load.attr, 344 &powersave_bias.attr, 345 &io_is_busy.attr, 346 NULL 347 }; 348 349 /************************** sysfs end ************************/ 350 351 static struct policy_dbs_info *od_alloc(void) 352 { 353 struct od_policy_dbs_info *dbs_info; 354 355 dbs_info = kzalloc(sizeof(*dbs_info), GFP_KERNEL); 356 return dbs_info ? &dbs_info->policy_dbs : NULL; 357 } 358 359 static void od_free(struct policy_dbs_info *policy_dbs) 360 { 361 kfree(to_dbs_info(policy_dbs)); 362 } 363 364 static int od_init(struct dbs_data *dbs_data, bool notify) 365 { 366 struct od_dbs_tuners *tuners; 367 u64 idle_time; 368 int cpu; 369 370 tuners = kzalloc(sizeof(*tuners), GFP_KERNEL); 371 if (!tuners) { 372 pr_err("%s: kzalloc failed\n", __func__); 373 return -ENOMEM; 374 } 375 376 cpu = get_cpu(); 377 idle_time = get_cpu_idle_time_us(cpu, NULL); 378 put_cpu(); 379 if (idle_time != -1ULL) { 380 /* Idle micro accounting is supported. Use finer thresholds */ 381 dbs_data->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD; 382 /* 383 * In nohz/micro accounting case we set the minimum frequency 384 * not depending on HZ, but fixed (very low). The deferred 385 * timer might skip some samples if idle/sleeping as needed. 386 */ 387 dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE; 388 } else { 389 dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD; 390 391 /* For correct statistics, we need 10 ticks for each measure */ 392 dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO * 393 jiffies_to_usecs(10); 394 } 395 396 dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR; 397 dbs_data->ignore_nice_load = 0; 398 tuners->powersave_bias = default_powersave_bias; 399 dbs_data->io_is_busy = should_io_be_busy(); 400 401 dbs_data->tuners = tuners; 402 return 0; 403 } 404 405 static void od_exit(struct dbs_data *dbs_data, bool notify) 406 { 407 kfree(dbs_data->tuners); 408 } 409 410 static void od_start(struct cpufreq_policy *policy) 411 { 412 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); 413 414 dbs_info->sample_type = OD_NORMAL_SAMPLE; 415 ondemand_powersave_bias_init(policy); 416 } 417 418 static struct od_ops od_ops = { 419 .powersave_bias_target = generic_powersave_bias_target, 420 }; 421 422 static struct dbs_governor od_dbs_gov = { 423 .gov = { 424 .name = "ondemand", 425 .governor = cpufreq_governor_dbs, 426 .max_transition_latency = TRANSITION_LATENCY_LIMIT, 427 .owner = THIS_MODULE, 428 }, 429 .kobj_type = { .default_attrs = od_attributes }, 430 .gov_dbs_timer = od_dbs_timer, 431 .alloc = od_alloc, 432 .free = od_free, 433 .init = od_init, 434 .exit = od_exit, 435 .start = od_start, 436 }; 437 438 #define CPU_FREQ_GOV_ONDEMAND (&od_dbs_gov.gov) 439 440 static void od_set_powersave_bias(unsigned int powersave_bias) 441 { 442 unsigned int cpu; 443 cpumask_t done; 444 445 default_powersave_bias = powersave_bias; 446 cpumask_clear(&done); 447 448 get_online_cpus(); 449 for_each_online_cpu(cpu) { 450 struct cpufreq_policy *policy; 451 struct policy_dbs_info *policy_dbs; 452 struct dbs_data *dbs_data; 453 struct od_dbs_tuners *od_tuners; 454 455 if (cpumask_test_cpu(cpu, &done)) 456 continue; 457 458 policy = cpufreq_cpu_get_raw(cpu); 459 if (!policy || policy->governor != CPU_FREQ_GOV_ONDEMAND) 460 continue; 461 462 policy_dbs = policy->governor_data; 463 if (!policy_dbs) 464 continue; 465 466 cpumask_or(&done, &done, policy->cpus); 467 468 dbs_data = policy_dbs->dbs_data; 469 od_tuners = dbs_data->tuners; 470 od_tuners->powersave_bias = default_powersave_bias; 471 } 472 put_online_cpus(); 473 } 474 475 void od_register_powersave_bias_handler(unsigned int (*f) 476 (struct cpufreq_policy *, unsigned int, unsigned int), 477 unsigned int powersave_bias) 478 { 479 od_ops.powersave_bias_target = f; 480 od_set_powersave_bias(powersave_bias); 481 } 482 EXPORT_SYMBOL_GPL(od_register_powersave_bias_handler); 483 484 void od_unregister_powersave_bias_handler(void) 485 { 486 od_ops.powersave_bias_target = generic_powersave_bias_target; 487 od_set_powersave_bias(0); 488 } 489 EXPORT_SYMBOL_GPL(od_unregister_powersave_bias_handler); 490 491 static int __init cpufreq_gov_dbs_init(void) 492 { 493 return cpufreq_register_governor(CPU_FREQ_GOV_ONDEMAND); 494 } 495 496 static void __exit cpufreq_gov_dbs_exit(void) 497 { 498 cpufreq_unregister_governor(CPU_FREQ_GOV_ONDEMAND); 499 } 500 501 MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>"); 502 MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>"); 503 MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for " 504 "Low Latency Frequency Transition capable processors"); 505 MODULE_LICENSE("GPL"); 506 507 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND 508 struct cpufreq_governor *cpufreq_default_governor(void) 509 { 510 return CPU_FREQ_GOV_ONDEMAND; 511 } 512 513 fs_initcall(cpufreq_gov_dbs_init); 514 #else 515 module_init(cpufreq_gov_dbs_init); 516 #endif 517 module_exit(cpufreq_gov_dbs_exit); 518