1 /* 2 * drivers/cpufreq/cpufreq_ondemand.c 3 * 4 * Copyright (C) 2001 Russell King 5 * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>. 6 * Jun Nakajima <jun.nakajima@intel.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License version 2 as 10 * published by the Free Software Foundation. 11 */ 12 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/cpu.h> 16 #include <linux/percpu-defs.h> 17 #include <linux/slab.h> 18 #include <linux/tick.h> 19 20 #include "cpufreq_ondemand.h" 21 22 /* On-demand governor macros */ 23 #define DEF_FREQUENCY_UP_THRESHOLD (80) 24 #define DEF_SAMPLING_DOWN_FACTOR (1) 25 #define MAX_SAMPLING_DOWN_FACTOR (100000) 26 #define MICRO_FREQUENCY_UP_THRESHOLD (95) 27 #define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000) 28 #define MIN_FREQUENCY_UP_THRESHOLD (11) 29 #define MAX_FREQUENCY_UP_THRESHOLD (100) 30 31 static struct od_ops od_ops; 32 33 static unsigned int default_powersave_bias; 34 35 /* 36 * Not all CPUs want IO time to be accounted as busy; this depends on how 37 * efficient idling at a higher frequency/voltage is. 38 * Pavel Machek says this is not so for various generations of AMD and old 39 * Intel systems. 40 * Mike Chan (android.com) claims this is also not true for ARM. 41 * Because of this, whitelist specific known (series) of CPUs by default, and 42 * leave all others up to the user. 43 */ 44 static int should_io_be_busy(void) 45 { 46 #if defined(CONFIG_X86) 47 /* 48 * For Intel, Core 2 (model 15) and later have an efficient idle. 49 */ 50 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && 51 boot_cpu_data.x86 == 6 && 52 boot_cpu_data.x86_model >= 15) 53 return 1; 54 #endif 55 return 0; 56 } 57 58 /* 59 * Find right freq to be set now with powersave_bias on. 60 * Returns the freq_hi to be used right now and will set freq_hi_delay_us, 61 * freq_lo, and freq_lo_delay_us in percpu area for averaging freqs. 62 */ 63 static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy, 64 unsigned int freq_next, unsigned int relation) 65 { 66 unsigned int freq_req, freq_reduc, freq_avg; 67 unsigned int freq_hi, freq_lo; 68 unsigned int index; 69 unsigned int delay_hi_us; 70 struct policy_dbs_info *policy_dbs = policy->governor_data; 71 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); 72 struct dbs_data *dbs_data = policy_dbs->dbs_data; 73 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 74 struct cpufreq_frequency_table *freq_table = policy->freq_table; 75 76 if (!freq_table) { 77 dbs_info->freq_lo = 0; 78 dbs_info->freq_lo_delay_us = 0; 79 return freq_next; 80 } 81 82 index = cpufreq_frequency_table_target(policy, freq_next, relation); 83 freq_req = freq_table[index].frequency; 84 freq_reduc = freq_req * od_tuners->powersave_bias / 1000; 85 freq_avg = freq_req - freq_reduc; 86 87 /* Find freq bounds for freq_avg in freq_table */ 88 index = cpufreq_table_find_index_h(policy, freq_avg); 89 freq_lo = freq_table[index].frequency; 90 index = cpufreq_table_find_index_l(policy, freq_avg); 91 freq_hi = freq_table[index].frequency; 92 93 /* Find out how long we have to be in hi and lo freqs */ 94 if (freq_hi == freq_lo) { 95 dbs_info->freq_lo = 0; 96 dbs_info->freq_lo_delay_us = 0; 97 return freq_lo; 98 } 99 delay_hi_us = (freq_avg - freq_lo) * dbs_data->sampling_rate; 100 delay_hi_us += (freq_hi - freq_lo) / 2; 101 delay_hi_us /= freq_hi - freq_lo; 102 dbs_info->freq_hi_delay_us = delay_hi_us; 103 dbs_info->freq_lo = freq_lo; 104 dbs_info->freq_lo_delay_us = dbs_data->sampling_rate - delay_hi_us; 105 return freq_hi; 106 } 107 108 static void ondemand_powersave_bias_init(struct cpufreq_policy *policy) 109 { 110 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); 111 112 dbs_info->freq_lo = 0; 113 } 114 115 static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq) 116 { 117 struct policy_dbs_info *policy_dbs = policy->governor_data; 118 struct dbs_data *dbs_data = policy_dbs->dbs_data; 119 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 120 121 if (od_tuners->powersave_bias) 122 freq = od_ops.powersave_bias_target(policy, freq, 123 CPUFREQ_RELATION_H); 124 else if (policy->cur == policy->max) 125 return; 126 127 __cpufreq_driver_target(policy, freq, od_tuners->powersave_bias ? 128 CPUFREQ_RELATION_L : CPUFREQ_RELATION_H); 129 } 130 131 /* 132 * Every sampling_rate, we check, if current idle time is less than 20% 133 * (default), then we try to increase frequency. Else, we adjust the frequency 134 * proportional to load. 135 */ 136 static void od_update(struct cpufreq_policy *policy) 137 { 138 struct policy_dbs_info *policy_dbs = policy->governor_data; 139 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); 140 struct dbs_data *dbs_data = policy_dbs->dbs_data; 141 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 142 unsigned int load = dbs_update(policy); 143 144 dbs_info->freq_lo = 0; 145 146 /* Check for frequency increase */ 147 if (load > dbs_data->up_threshold) { 148 /* If switching to max speed, apply sampling_down_factor */ 149 if (policy->cur < policy->max) 150 policy_dbs->rate_mult = dbs_data->sampling_down_factor; 151 dbs_freq_increase(policy, policy->max); 152 } else { 153 /* Calculate the next frequency proportional to load */ 154 unsigned int freq_next, min_f, max_f; 155 156 min_f = policy->cpuinfo.min_freq; 157 max_f = policy->cpuinfo.max_freq; 158 freq_next = min_f + load * (max_f - min_f) / 100; 159 160 /* No longer fully busy, reset rate_mult */ 161 policy_dbs->rate_mult = 1; 162 163 if (od_tuners->powersave_bias) 164 freq_next = od_ops.powersave_bias_target(policy, 165 freq_next, 166 CPUFREQ_RELATION_L); 167 168 __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_C); 169 } 170 } 171 172 static unsigned int od_dbs_update(struct cpufreq_policy *policy) 173 { 174 struct policy_dbs_info *policy_dbs = policy->governor_data; 175 struct dbs_data *dbs_data = policy_dbs->dbs_data; 176 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); 177 int sample_type = dbs_info->sample_type; 178 179 /* Common NORMAL_SAMPLE setup */ 180 dbs_info->sample_type = OD_NORMAL_SAMPLE; 181 /* 182 * OD_SUB_SAMPLE doesn't make sense if sample_delay_ns is 0, so ignore 183 * it then. 184 */ 185 if (sample_type == OD_SUB_SAMPLE && policy_dbs->sample_delay_ns > 0) { 186 __cpufreq_driver_target(policy, dbs_info->freq_lo, 187 CPUFREQ_RELATION_H); 188 return dbs_info->freq_lo_delay_us; 189 } 190 191 od_update(policy); 192 193 if (dbs_info->freq_lo) { 194 /* Setup SUB_SAMPLE */ 195 dbs_info->sample_type = OD_SUB_SAMPLE; 196 return dbs_info->freq_hi_delay_us; 197 } 198 199 return dbs_data->sampling_rate * policy_dbs->rate_mult; 200 } 201 202 /************************** sysfs interface ************************/ 203 static struct dbs_governor od_dbs_gov; 204 205 static ssize_t store_io_is_busy(struct gov_attr_set *attr_set, const char *buf, 206 size_t count) 207 { 208 struct dbs_data *dbs_data = to_dbs_data(attr_set); 209 unsigned int input; 210 int ret; 211 212 ret = sscanf(buf, "%u", &input); 213 if (ret != 1) 214 return -EINVAL; 215 dbs_data->io_is_busy = !!input; 216 217 /* we need to re-evaluate prev_cpu_idle */ 218 gov_update_cpu_data(dbs_data); 219 220 return count; 221 } 222 223 static ssize_t store_up_threshold(struct gov_attr_set *attr_set, 224 const char *buf, size_t count) 225 { 226 struct dbs_data *dbs_data = to_dbs_data(attr_set); 227 unsigned int input; 228 int ret; 229 ret = sscanf(buf, "%u", &input); 230 231 if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD || 232 input < MIN_FREQUENCY_UP_THRESHOLD) { 233 return -EINVAL; 234 } 235 236 dbs_data->up_threshold = input; 237 return count; 238 } 239 240 static ssize_t store_sampling_down_factor(struct gov_attr_set *attr_set, 241 const char *buf, size_t count) 242 { 243 struct dbs_data *dbs_data = to_dbs_data(attr_set); 244 struct policy_dbs_info *policy_dbs; 245 unsigned int input; 246 int ret; 247 ret = sscanf(buf, "%u", &input); 248 249 if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1) 250 return -EINVAL; 251 252 dbs_data->sampling_down_factor = input; 253 254 /* Reset down sampling multiplier in case it was active */ 255 list_for_each_entry(policy_dbs, &attr_set->policy_list, list) { 256 /* 257 * Doing this without locking might lead to using different 258 * rate_mult values in od_update() and od_dbs_update(). 259 */ 260 mutex_lock(&policy_dbs->update_mutex); 261 policy_dbs->rate_mult = 1; 262 mutex_unlock(&policy_dbs->update_mutex); 263 } 264 265 return count; 266 } 267 268 static ssize_t store_ignore_nice_load(struct gov_attr_set *attr_set, 269 const char *buf, size_t count) 270 { 271 struct dbs_data *dbs_data = to_dbs_data(attr_set); 272 unsigned int input; 273 int ret; 274 275 ret = sscanf(buf, "%u", &input); 276 if (ret != 1) 277 return -EINVAL; 278 279 if (input > 1) 280 input = 1; 281 282 if (input == dbs_data->ignore_nice_load) { /* nothing to do */ 283 return count; 284 } 285 dbs_data->ignore_nice_load = input; 286 287 /* we need to re-evaluate prev_cpu_idle */ 288 gov_update_cpu_data(dbs_data); 289 290 return count; 291 } 292 293 static ssize_t store_powersave_bias(struct gov_attr_set *attr_set, 294 const char *buf, size_t count) 295 { 296 struct dbs_data *dbs_data = to_dbs_data(attr_set); 297 struct od_dbs_tuners *od_tuners = dbs_data->tuners; 298 struct policy_dbs_info *policy_dbs; 299 unsigned int input; 300 int ret; 301 ret = sscanf(buf, "%u", &input); 302 303 if (ret != 1) 304 return -EINVAL; 305 306 if (input > 1000) 307 input = 1000; 308 309 od_tuners->powersave_bias = input; 310 311 list_for_each_entry(policy_dbs, &attr_set->policy_list, list) 312 ondemand_powersave_bias_init(policy_dbs->policy); 313 314 return count; 315 } 316 317 gov_show_one_common(sampling_rate); 318 gov_show_one_common(up_threshold); 319 gov_show_one_common(sampling_down_factor); 320 gov_show_one_common(ignore_nice_load); 321 gov_show_one_common(min_sampling_rate); 322 gov_show_one_common(io_is_busy); 323 gov_show_one(od, powersave_bias); 324 325 gov_attr_rw(sampling_rate); 326 gov_attr_rw(io_is_busy); 327 gov_attr_rw(up_threshold); 328 gov_attr_rw(sampling_down_factor); 329 gov_attr_rw(ignore_nice_load); 330 gov_attr_rw(powersave_bias); 331 gov_attr_ro(min_sampling_rate); 332 333 static struct attribute *od_attributes[] = { 334 &min_sampling_rate.attr, 335 &sampling_rate.attr, 336 &up_threshold.attr, 337 &sampling_down_factor.attr, 338 &ignore_nice_load.attr, 339 &powersave_bias.attr, 340 &io_is_busy.attr, 341 NULL 342 }; 343 344 /************************** sysfs end ************************/ 345 346 static struct policy_dbs_info *od_alloc(void) 347 { 348 struct od_policy_dbs_info *dbs_info; 349 350 dbs_info = kzalloc(sizeof(*dbs_info), GFP_KERNEL); 351 return dbs_info ? &dbs_info->policy_dbs : NULL; 352 } 353 354 static void od_free(struct policy_dbs_info *policy_dbs) 355 { 356 kfree(to_dbs_info(policy_dbs)); 357 } 358 359 static int od_init(struct dbs_data *dbs_data) 360 { 361 struct od_dbs_tuners *tuners; 362 u64 idle_time; 363 int cpu; 364 365 tuners = kzalloc(sizeof(*tuners), GFP_KERNEL); 366 if (!tuners) 367 return -ENOMEM; 368 369 cpu = get_cpu(); 370 idle_time = get_cpu_idle_time_us(cpu, NULL); 371 put_cpu(); 372 if (idle_time != -1ULL) { 373 /* Idle micro accounting is supported. Use finer thresholds */ 374 dbs_data->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD; 375 /* 376 * In nohz/micro accounting case we set the minimum frequency 377 * not depending on HZ, but fixed (very low). 378 */ 379 dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE; 380 } else { 381 dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD; 382 383 /* For correct statistics, we need 10 ticks for each measure */ 384 dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO * 385 jiffies_to_usecs(10); 386 } 387 388 dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR; 389 dbs_data->ignore_nice_load = 0; 390 tuners->powersave_bias = default_powersave_bias; 391 dbs_data->io_is_busy = should_io_be_busy(); 392 393 dbs_data->tuners = tuners; 394 return 0; 395 } 396 397 static void od_exit(struct dbs_data *dbs_data) 398 { 399 kfree(dbs_data->tuners); 400 } 401 402 static void od_start(struct cpufreq_policy *policy) 403 { 404 struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data); 405 406 dbs_info->sample_type = OD_NORMAL_SAMPLE; 407 ondemand_powersave_bias_init(policy); 408 } 409 410 static struct od_ops od_ops = { 411 .powersave_bias_target = generic_powersave_bias_target, 412 }; 413 414 static struct dbs_governor od_dbs_gov = { 415 .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("ondemand"), 416 .kobj_type = { .default_attrs = od_attributes }, 417 .gov_dbs_update = od_dbs_update, 418 .alloc = od_alloc, 419 .free = od_free, 420 .init = od_init, 421 .exit = od_exit, 422 .start = od_start, 423 }; 424 425 #define CPU_FREQ_GOV_ONDEMAND (&od_dbs_gov.gov) 426 427 static void od_set_powersave_bias(unsigned int powersave_bias) 428 { 429 unsigned int cpu; 430 cpumask_t done; 431 432 default_powersave_bias = powersave_bias; 433 cpumask_clear(&done); 434 435 get_online_cpus(); 436 for_each_online_cpu(cpu) { 437 struct cpufreq_policy *policy; 438 struct policy_dbs_info *policy_dbs; 439 struct dbs_data *dbs_data; 440 struct od_dbs_tuners *od_tuners; 441 442 if (cpumask_test_cpu(cpu, &done)) 443 continue; 444 445 policy = cpufreq_cpu_get_raw(cpu); 446 if (!policy || policy->governor != CPU_FREQ_GOV_ONDEMAND) 447 continue; 448 449 policy_dbs = policy->governor_data; 450 if (!policy_dbs) 451 continue; 452 453 cpumask_or(&done, &done, policy->cpus); 454 455 dbs_data = policy_dbs->dbs_data; 456 od_tuners = dbs_data->tuners; 457 od_tuners->powersave_bias = default_powersave_bias; 458 } 459 put_online_cpus(); 460 } 461 462 void od_register_powersave_bias_handler(unsigned int (*f) 463 (struct cpufreq_policy *, unsigned int, unsigned int), 464 unsigned int powersave_bias) 465 { 466 od_ops.powersave_bias_target = f; 467 od_set_powersave_bias(powersave_bias); 468 } 469 EXPORT_SYMBOL_GPL(od_register_powersave_bias_handler); 470 471 void od_unregister_powersave_bias_handler(void) 472 { 473 od_ops.powersave_bias_target = generic_powersave_bias_target; 474 od_set_powersave_bias(0); 475 } 476 EXPORT_SYMBOL_GPL(od_unregister_powersave_bias_handler); 477 478 static int __init cpufreq_gov_dbs_init(void) 479 { 480 return cpufreq_register_governor(CPU_FREQ_GOV_ONDEMAND); 481 } 482 483 static void __exit cpufreq_gov_dbs_exit(void) 484 { 485 cpufreq_unregister_governor(CPU_FREQ_GOV_ONDEMAND); 486 } 487 488 MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>"); 489 MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>"); 490 MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for " 491 "Low Latency Frequency Transition capable processors"); 492 MODULE_LICENSE("GPL"); 493 494 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND 495 struct cpufreq_governor *cpufreq_default_governor(void) 496 { 497 return CPU_FREQ_GOV_ONDEMAND; 498 } 499 500 fs_initcall(cpufreq_gov_dbs_init); 501 #else 502 module_init(cpufreq_gov_dbs_init); 503 #endif 504 module_exit(cpufreq_gov_dbs_exit); 505