1 /* 2 * linux/drivers/cpufreq/cpufreq.c 3 * 4 * Copyright (C) 2001 Russell King 5 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> 6 * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org> 7 * 8 * Oct 2005 - Ashok Raj <ashok.raj@intel.com> 9 * Added handling for CPU hotplug 10 * Feb 2006 - Jacob Shin <jacob.shin@amd.com> 11 * Fix handling for CPU hotplug -- affected CPUs 12 * 13 * This program is free software; you can redistribute it and/or modify 14 * it under the terms of the GNU General Public License version 2 as 15 * published by the Free Software Foundation. 16 */ 17 18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 19 20 #include <linux/cpu.h> 21 #include <linux/cpufreq.h> 22 #include <linux/delay.h> 23 #include <linux/device.h> 24 #include <linux/init.h> 25 #include <linux/kernel_stat.h> 26 #include <linux/module.h> 27 #include <linux/mutex.h> 28 #include <linux/slab.h> 29 #include <linux/suspend.h> 30 #include <linux/syscore_ops.h> 31 #include <linux/tick.h> 32 #include <trace/events/power.h> 33 34 static LIST_HEAD(cpufreq_policy_list); 35 36 static inline bool policy_is_inactive(struct cpufreq_policy *policy) 37 { 38 return cpumask_empty(policy->cpus); 39 } 40 41 static bool suitable_policy(struct cpufreq_policy *policy, bool active) 42 { 43 return active == !policy_is_inactive(policy); 44 } 45 46 /* Finds Next Acive/Inactive policy */ 47 static struct cpufreq_policy *next_policy(struct cpufreq_policy *policy, 48 bool active) 49 { 50 do { 51 policy = list_next_entry(policy, policy_list); 52 53 /* No more policies in the list */ 54 if (&policy->policy_list == &cpufreq_policy_list) 55 return NULL; 56 } while (!suitable_policy(policy, active)); 57 58 return policy; 59 } 60 61 static struct cpufreq_policy *first_policy(bool active) 62 { 63 struct cpufreq_policy *policy; 64 65 /* No policies in the list */ 66 if (list_empty(&cpufreq_policy_list)) 67 return NULL; 68 69 policy = list_first_entry(&cpufreq_policy_list, typeof(*policy), 70 policy_list); 71 72 if (!suitable_policy(policy, active)) 73 policy = next_policy(policy, active); 74 75 return policy; 76 } 77 78 /* Macros to iterate over CPU policies */ 79 #define for_each_suitable_policy(__policy, __active) \ 80 for (__policy = first_policy(__active); \ 81 __policy; \ 82 __policy = next_policy(__policy, __active)) 83 84 #define for_each_active_policy(__policy) \ 85 for_each_suitable_policy(__policy, true) 86 #define for_each_inactive_policy(__policy) \ 87 for_each_suitable_policy(__policy, false) 88 89 #define for_each_policy(__policy) \ 90 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) 91 92 /* Iterate over governors */ 93 static LIST_HEAD(cpufreq_governor_list); 94 #define for_each_governor(__governor) \ 95 list_for_each_entry(__governor, &cpufreq_governor_list, governor_list) 96 97 /** 98 * The "cpufreq driver" - the arch- or hardware-dependent low 99 * level driver of CPUFreq support, and its spinlock. This lock 100 * also protects the cpufreq_cpu_data array. 101 */ 102 static struct cpufreq_driver *cpufreq_driver; 103 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data); 104 static DEFINE_RWLOCK(cpufreq_driver_lock); 105 DEFINE_MUTEX(cpufreq_governor_lock); 106 107 /* Flag to suspend/resume CPUFreq governors */ 108 static bool cpufreq_suspended; 109 110 static inline bool has_target(void) 111 { 112 return cpufreq_driver->target_index || cpufreq_driver->target; 113 } 114 115 /* 116 * rwsem to guarantee that cpufreq driver module doesn't unload during critical 117 * sections 118 */ 119 static DECLARE_RWSEM(cpufreq_rwsem); 120 121 /* internal prototypes */ 122 static int __cpufreq_governor(struct cpufreq_policy *policy, 123 unsigned int event); 124 static unsigned int __cpufreq_get(struct cpufreq_policy *policy); 125 static void handle_update(struct work_struct *work); 126 127 /** 128 * Two notifier lists: the "policy" list is involved in the 129 * validation process for a new CPU frequency policy; the 130 * "transition" list for kernel code that needs to handle 131 * changes to devices when the CPU clock speed changes. 132 * The mutex locks both lists. 133 */ 134 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list); 135 static struct srcu_notifier_head cpufreq_transition_notifier_list; 136 137 static bool init_cpufreq_transition_notifier_list_called; 138 static int __init init_cpufreq_transition_notifier_list(void) 139 { 140 srcu_init_notifier_head(&cpufreq_transition_notifier_list); 141 init_cpufreq_transition_notifier_list_called = true; 142 return 0; 143 } 144 pure_initcall(init_cpufreq_transition_notifier_list); 145 146 static int off __read_mostly; 147 static int cpufreq_disabled(void) 148 { 149 return off; 150 } 151 void disable_cpufreq(void) 152 { 153 off = 1; 154 } 155 static DEFINE_MUTEX(cpufreq_governor_mutex); 156 157 bool have_governor_per_policy(void) 158 { 159 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY); 160 } 161 EXPORT_SYMBOL_GPL(have_governor_per_policy); 162 163 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) 164 { 165 if (have_governor_per_policy()) 166 return &policy->kobj; 167 else 168 return cpufreq_global_kobject; 169 } 170 EXPORT_SYMBOL_GPL(get_governor_parent_kobj); 171 172 struct cpufreq_frequency_table *cpufreq_frequency_get_table(unsigned int cpu) 173 { 174 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 175 176 return policy && !policy_is_inactive(policy) ? 177 policy->freq_table : NULL; 178 } 179 EXPORT_SYMBOL_GPL(cpufreq_frequency_get_table); 180 181 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) 182 { 183 u64 idle_time; 184 u64 cur_wall_time; 185 u64 busy_time; 186 187 cur_wall_time = jiffies64_to_cputime64(get_jiffies_64()); 188 189 busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; 190 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; 191 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; 192 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; 193 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; 194 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; 195 196 idle_time = cur_wall_time - busy_time; 197 if (wall) 198 *wall = cputime_to_usecs(cur_wall_time); 199 200 return cputime_to_usecs(idle_time); 201 } 202 203 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) 204 { 205 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL); 206 207 if (idle_time == -1ULL) 208 return get_cpu_idle_time_jiffy(cpu, wall); 209 else if (!io_busy) 210 idle_time += get_cpu_iowait_time_us(cpu, wall); 211 212 return idle_time; 213 } 214 EXPORT_SYMBOL_GPL(get_cpu_idle_time); 215 216 /* 217 * This is a generic cpufreq init() routine which can be used by cpufreq 218 * drivers of SMP systems. It will do following: 219 * - validate & show freq table passed 220 * - set policies transition latency 221 * - policy->cpus with all possible CPUs 222 */ 223 int cpufreq_generic_init(struct cpufreq_policy *policy, 224 struct cpufreq_frequency_table *table, 225 unsigned int transition_latency) 226 { 227 int ret; 228 229 ret = cpufreq_table_validate_and_show(policy, table); 230 if (ret) { 231 pr_err("%s: invalid frequency table: %d\n", __func__, ret); 232 return ret; 233 } 234 235 policy->cpuinfo.transition_latency = transition_latency; 236 237 /* 238 * The driver only supports the SMP configuration where all processors 239 * share the clock and voltage and clock. 240 */ 241 cpumask_setall(policy->cpus); 242 243 return 0; 244 } 245 EXPORT_SYMBOL_GPL(cpufreq_generic_init); 246 247 /* Only for cpufreq core internal use */ 248 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu) 249 { 250 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 251 252 return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL; 253 } 254 255 unsigned int cpufreq_generic_get(unsigned int cpu) 256 { 257 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); 258 259 if (!policy || IS_ERR(policy->clk)) { 260 pr_err("%s: No %s associated to cpu: %d\n", 261 __func__, policy ? "clk" : "policy", cpu); 262 return 0; 263 } 264 265 return clk_get_rate(policy->clk) / 1000; 266 } 267 EXPORT_SYMBOL_GPL(cpufreq_generic_get); 268 269 /** 270 * cpufreq_cpu_get: returns policy for a cpu and marks it busy. 271 * 272 * @cpu: cpu to find policy for. 273 * 274 * This returns policy for 'cpu', returns NULL if it doesn't exist. 275 * It also increments the kobject reference count to mark it busy and so would 276 * require a corresponding call to cpufreq_cpu_put() to decrement it back. 277 * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be 278 * freed as that depends on the kobj count. 279 * 280 * It also takes a read-lock of 'cpufreq_rwsem' and doesn't put it back if a 281 * valid policy is found. This is done to make sure the driver doesn't get 282 * unregistered while the policy is being used. 283 * 284 * Return: A valid policy on success, otherwise NULL on failure. 285 */ 286 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) 287 { 288 struct cpufreq_policy *policy = NULL; 289 unsigned long flags; 290 291 if (WARN_ON(cpu >= nr_cpu_ids)) 292 return NULL; 293 294 if (!down_read_trylock(&cpufreq_rwsem)) 295 return NULL; 296 297 /* get the cpufreq driver */ 298 read_lock_irqsave(&cpufreq_driver_lock, flags); 299 300 if (cpufreq_driver) { 301 /* get the CPU */ 302 policy = cpufreq_cpu_get_raw(cpu); 303 if (policy) 304 kobject_get(&policy->kobj); 305 } 306 307 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 308 309 if (!policy) 310 up_read(&cpufreq_rwsem); 311 312 return policy; 313 } 314 EXPORT_SYMBOL_GPL(cpufreq_cpu_get); 315 316 /** 317 * cpufreq_cpu_put: Decrements the usage count of a policy 318 * 319 * @policy: policy earlier returned by cpufreq_cpu_get(). 320 * 321 * This decrements the kobject reference count incremented earlier by calling 322 * cpufreq_cpu_get(). 323 * 324 * It also drops the read-lock of 'cpufreq_rwsem' taken at cpufreq_cpu_get(). 325 */ 326 void cpufreq_cpu_put(struct cpufreq_policy *policy) 327 { 328 kobject_put(&policy->kobj); 329 up_read(&cpufreq_rwsem); 330 } 331 EXPORT_SYMBOL_GPL(cpufreq_cpu_put); 332 333 /********************************************************************* 334 * EXTERNALLY AFFECTING FREQUENCY CHANGES * 335 *********************************************************************/ 336 337 /** 338 * adjust_jiffies - adjust the system "loops_per_jiffy" 339 * 340 * This function alters the system "loops_per_jiffy" for the clock 341 * speed change. Note that loops_per_jiffy cannot be updated on SMP 342 * systems as each CPU might be scaled differently. So, use the arch 343 * per-CPU loops_per_jiffy value wherever possible. 344 */ 345 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) 346 { 347 #ifndef CONFIG_SMP 348 static unsigned long l_p_j_ref; 349 static unsigned int l_p_j_ref_freq; 350 351 if (ci->flags & CPUFREQ_CONST_LOOPS) 352 return; 353 354 if (!l_p_j_ref_freq) { 355 l_p_j_ref = loops_per_jiffy; 356 l_p_j_ref_freq = ci->old; 357 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n", 358 l_p_j_ref, l_p_j_ref_freq); 359 } 360 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) { 361 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq, 362 ci->new); 363 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n", 364 loops_per_jiffy, ci->new); 365 } 366 #endif 367 } 368 369 static void __cpufreq_notify_transition(struct cpufreq_policy *policy, 370 struct cpufreq_freqs *freqs, unsigned int state) 371 { 372 BUG_ON(irqs_disabled()); 373 374 if (cpufreq_disabled()) 375 return; 376 377 freqs->flags = cpufreq_driver->flags; 378 pr_debug("notification %u of frequency transition to %u kHz\n", 379 state, freqs->new); 380 381 switch (state) { 382 383 case CPUFREQ_PRECHANGE: 384 /* detect if the driver reported a value as "old frequency" 385 * which is not equal to what the cpufreq core thinks is 386 * "old frequency". 387 */ 388 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) { 389 if ((policy) && (policy->cpu == freqs->cpu) && 390 (policy->cur) && (policy->cur != freqs->old)) { 391 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n", 392 freqs->old, policy->cur); 393 freqs->old = policy->cur; 394 } 395 } 396 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 397 CPUFREQ_PRECHANGE, freqs); 398 adjust_jiffies(CPUFREQ_PRECHANGE, freqs); 399 break; 400 401 case CPUFREQ_POSTCHANGE: 402 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs); 403 pr_debug("FREQ: %lu - CPU: %lu\n", 404 (unsigned long)freqs->new, (unsigned long)freqs->cpu); 405 trace_cpu_frequency(freqs->new, freqs->cpu); 406 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 407 CPUFREQ_POSTCHANGE, freqs); 408 if (likely(policy) && likely(policy->cpu == freqs->cpu)) 409 policy->cur = freqs->new; 410 break; 411 } 412 } 413 414 /** 415 * cpufreq_notify_transition - call notifier chain and adjust_jiffies 416 * on frequency transition. 417 * 418 * This function calls the transition notifiers and the "adjust_jiffies" 419 * function. It is called twice on all CPU frequency changes that have 420 * external effects. 421 */ 422 static void cpufreq_notify_transition(struct cpufreq_policy *policy, 423 struct cpufreq_freqs *freqs, unsigned int state) 424 { 425 for_each_cpu(freqs->cpu, policy->cpus) 426 __cpufreq_notify_transition(policy, freqs, state); 427 } 428 429 /* Do post notifications when there are chances that transition has failed */ 430 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy, 431 struct cpufreq_freqs *freqs, int transition_failed) 432 { 433 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 434 if (!transition_failed) 435 return; 436 437 swap(freqs->old, freqs->new); 438 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 439 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 440 } 441 442 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy, 443 struct cpufreq_freqs *freqs) 444 { 445 446 /* 447 * Catch double invocations of _begin() which lead to self-deadlock. 448 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core 449 * doesn't invoke _begin() on their behalf, and hence the chances of 450 * double invocations are very low. Moreover, there are scenarios 451 * where these checks can emit false-positive warnings in these 452 * drivers; so we avoid that by skipping them altogether. 453 */ 454 WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION) 455 && current == policy->transition_task); 456 457 wait: 458 wait_event(policy->transition_wait, !policy->transition_ongoing); 459 460 spin_lock(&policy->transition_lock); 461 462 if (unlikely(policy->transition_ongoing)) { 463 spin_unlock(&policy->transition_lock); 464 goto wait; 465 } 466 467 policy->transition_ongoing = true; 468 policy->transition_task = current; 469 470 spin_unlock(&policy->transition_lock); 471 472 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 473 } 474 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin); 475 476 void cpufreq_freq_transition_end(struct cpufreq_policy *policy, 477 struct cpufreq_freqs *freqs, int transition_failed) 478 { 479 if (unlikely(WARN_ON(!policy->transition_ongoing))) 480 return; 481 482 cpufreq_notify_post_transition(policy, freqs, transition_failed); 483 484 policy->transition_ongoing = false; 485 policy->transition_task = NULL; 486 487 wake_up(&policy->transition_wait); 488 } 489 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end); 490 491 492 /********************************************************************* 493 * SYSFS INTERFACE * 494 *********************************************************************/ 495 static ssize_t show_boost(struct kobject *kobj, 496 struct attribute *attr, char *buf) 497 { 498 return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled); 499 } 500 501 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr, 502 const char *buf, size_t count) 503 { 504 int ret, enable; 505 506 ret = sscanf(buf, "%d", &enable); 507 if (ret != 1 || enable < 0 || enable > 1) 508 return -EINVAL; 509 510 if (cpufreq_boost_trigger_state(enable)) { 511 pr_err("%s: Cannot %s BOOST!\n", 512 __func__, enable ? "enable" : "disable"); 513 return -EINVAL; 514 } 515 516 pr_debug("%s: cpufreq BOOST %s\n", 517 __func__, enable ? "enabled" : "disabled"); 518 519 return count; 520 } 521 define_one_global_rw(boost); 522 523 static struct cpufreq_governor *find_governor(const char *str_governor) 524 { 525 struct cpufreq_governor *t; 526 527 for_each_governor(t) 528 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN)) 529 return t; 530 531 return NULL; 532 } 533 534 /** 535 * cpufreq_parse_governor - parse a governor string 536 */ 537 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy, 538 struct cpufreq_governor **governor) 539 { 540 int err = -EINVAL; 541 542 if (!cpufreq_driver) 543 goto out; 544 545 if (cpufreq_driver->setpolicy) { 546 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) { 547 *policy = CPUFREQ_POLICY_PERFORMANCE; 548 err = 0; 549 } else if (!strncasecmp(str_governor, "powersave", 550 CPUFREQ_NAME_LEN)) { 551 *policy = CPUFREQ_POLICY_POWERSAVE; 552 err = 0; 553 } 554 } else { 555 struct cpufreq_governor *t; 556 557 mutex_lock(&cpufreq_governor_mutex); 558 559 t = find_governor(str_governor); 560 561 if (t == NULL) { 562 int ret; 563 564 mutex_unlock(&cpufreq_governor_mutex); 565 ret = request_module("cpufreq_%s", str_governor); 566 mutex_lock(&cpufreq_governor_mutex); 567 568 if (ret == 0) 569 t = find_governor(str_governor); 570 } 571 572 if (t != NULL) { 573 *governor = t; 574 err = 0; 575 } 576 577 mutex_unlock(&cpufreq_governor_mutex); 578 } 579 out: 580 return err; 581 } 582 583 /** 584 * cpufreq_per_cpu_attr_read() / show_##file_name() - 585 * print out cpufreq information 586 * 587 * Write out information from cpufreq_driver->policy[cpu]; object must be 588 * "unsigned int". 589 */ 590 591 #define show_one(file_name, object) \ 592 static ssize_t show_##file_name \ 593 (struct cpufreq_policy *policy, char *buf) \ 594 { \ 595 return sprintf(buf, "%u\n", policy->object); \ 596 } 597 598 show_one(cpuinfo_min_freq, cpuinfo.min_freq); 599 show_one(cpuinfo_max_freq, cpuinfo.max_freq); 600 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency); 601 show_one(scaling_min_freq, min); 602 show_one(scaling_max_freq, max); 603 604 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf) 605 { 606 ssize_t ret; 607 608 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) 609 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu)); 610 else 611 ret = sprintf(buf, "%u\n", policy->cur); 612 return ret; 613 } 614 615 static int cpufreq_set_policy(struct cpufreq_policy *policy, 616 struct cpufreq_policy *new_policy); 617 618 /** 619 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access 620 */ 621 #define store_one(file_name, object) \ 622 static ssize_t store_##file_name \ 623 (struct cpufreq_policy *policy, const char *buf, size_t count) \ 624 { \ 625 int ret, temp; \ 626 struct cpufreq_policy new_policy; \ 627 \ 628 ret = cpufreq_get_policy(&new_policy, policy->cpu); \ 629 if (ret) \ 630 return -EINVAL; \ 631 \ 632 ret = sscanf(buf, "%u", &new_policy.object); \ 633 if (ret != 1) \ 634 return -EINVAL; \ 635 \ 636 temp = new_policy.object; \ 637 ret = cpufreq_set_policy(policy, &new_policy); \ 638 if (!ret) \ 639 policy->user_policy.object = temp; \ 640 \ 641 return ret ? ret : count; \ 642 } 643 644 store_one(scaling_min_freq, min); 645 store_one(scaling_max_freq, max); 646 647 /** 648 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware 649 */ 650 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, 651 char *buf) 652 { 653 unsigned int cur_freq = __cpufreq_get(policy); 654 if (!cur_freq) 655 return sprintf(buf, "<unknown>"); 656 return sprintf(buf, "%u\n", cur_freq); 657 } 658 659 /** 660 * show_scaling_governor - show the current policy for the specified CPU 661 */ 662 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) 663 { 664 if (policy->policy == CPUFREQ_POLICY_POWERSAVE) 665 return sprintf(buf, "powersave\n"); 666 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) 667 return sprintf(buf, "performance\n"); 668 else if (policy->governor) 669 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", 670 policy->governor->name); 671 return -EINVAL; 672 } 673 674 /** 675 * store_scaling_governor - store policy for the specified CPU 676 */ 677 static ssize_t store_scaling_governor(struct cpufreq_policy *policy, 678 const char *buf, size_t count) 679 { 680 int ret; 681 char str_governor[16]; 682 struct cpufreq_policy new_policy; 683 684 ret = cpufreq_get_policy(&new_policy, policy->cpu); 685 if (ret) 686 return ret; 687 688 ret = sscanf(buf, "%15s", str_governor); 689 if (ret != 1) 690 return -EINVAL; 691 692 if (cpufreq_parse_governor(str_governor, &new_policy.policy, 693 &new_policy.governor)) 694 return -EINVAL; 695 696 ret = cpufreq_set_policy(policy, &new_policy); 697 698 policy->user_policy.policy = policy->policy; 699 policy->user_policy.governor = policy->governor; 700 701 if (ret) 702 return ret; 703 else 704 return count; 705 } 706 707 /** 708 * show_scaling_driver - show the cpufreq driver currently loaded 709 */ 710 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf) 711 { 712 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name); 713 } 714 715 /** 716 * show_scaling_available_governors - show the available CPUfreq governors 717 */ 718 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy, 719 char *buf) 720 { 721 ssize_t i = 0; 722 struct cpufreq_governor *t; 723 724 if (!has_target()) { 725 i += sprintf(buf, "performance powersave"); 726 goto out; 727 } 728 729 for_each_governor(t) { 730 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) 731 - (CPUFREQ_NAME_LEN + 2))) 732 goto out; 733 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name); 734 } 735 out: 736 i += sprintf(&buf[i], "\n"); 737 return i; 738 } 739 740 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf) 741 { 742 ssize_t i = 0; 743 unsigned int cpu; 744 745 for_each_cpu(cpu, mask) { 746 if (i) 747 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " "); 748 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu); 749 if (i >= (PAGE_SIZE - 5)) 750 break; 751 } 752 i += sprintf(&buf[i], "\n"); 753 return i; 754 } 755 EXPORT_SYMBOL_GPL(cpufreq_show_cpus); 756 757 /** 758 * show_related_cpus - show the CPUs affected by each transition even if 759 * hw coordination is in use 760 */ 761 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf) 762 { 763 return cpufreq_show_cpus(policy->related_cpus, buf); 764 } 765 766 /** 767 * show_affected_cpus - show the CPUs affected by each transition 768 */ 769 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf) 770 { 771 return cpufreq_show_cpus(policy->cpus, buf); 772 } 773 774 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy, 775 const char *buf, size_t count) 776 { 777 unsigned int freq = 0; 778 unsigned int ret; 779 780 if (!policy->governor || !policy->governor->store_setspeed) 781 return -EINVAL; 782 783 ret = sscanf(buf, "%u", &freq); 784 if (ret != 1) 785 return -EINVAL; 786 787 policy->governor->store_setspeed(policy, freq); 788 789 return count; 790 } 791 792 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf) 793 { 794 if (!policy->governor || !policy->governor->show_setspeed) 795 return sprintf(buf, "<unsupported>\n"); 796 797 return policy->governor->show_setspeed(policy, buf); 798 } 799 800 /** 801 * show_bios_limit - show the current cpufreq HW/BIOS limitation 802 */ 803 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf) 804 { 805 unsigned int limit; 806 int ret; 807 if (cpufreq_driver->bios_limit) { 808 ret = cpufreq_driver->bios_limit(policy->cpu, &limit); 809 if (!ret) 810 return sprintf(buf, "%u\n", limit); 811 } 812 return sprintf(buf, "%u\n", policy->cpuinfo.max_freq); 813 } 814 815 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400); 816 cpufreq_freq_attr_ro(cpuinfo_min_freq); 817 cpufreq_freq_attr_ro(cpuinfo_max_freq); 818 cpufreq_freq_attr_ro(cpuinfo_transition_latency); 819 cpufreq_freq_attr_ro(scaling_available_governors); 820 cpufreq_freq_attr_ro(scaling_driver); 821 cpufreq_freq_attr_ro(scaling_cur_freq); 822 cpufreq_freq_attr_ro(bios_limit); 823 cpufreq_freq_attr_ro(related_cpus); 824 cpufreq_freq_attr_ro(affected_cpus); 825 cpufreq_freq_attr_rw(scaling_min_freq); 826 cpufreq_freq_attr_rw(scaling_max_freq); 827 cpufreq_freq_attr_rw(scaling_governor); 828 cpufreq_freq_attr_rw(scaling_setspeed); 829 830 static struct attribute *default_attrs[] = { 831 &cpuinfo_min_freq.attr, 832 &cpuinfo_max_freq.attr, 833 &cpuinfo_transition_latency.attr, 834 &scaling_min_freq.attr, 835 &scaling_max_freq.attr, 836 &affected_cpus.attr, 837 &related_cpus.attr, 838 &scaling_governor.attr, 839 &scaling_driver.attr, 840 &scaling_available_governors.attr, 841 &scaling_setspeed.attr, 842 NULL 843 }; 844 845 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj) 846 #define to_attr(a) container_of(a, struct freq_attr, attr) 847 848 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) 849 { 850 struct cpufreq_policy *policy = to_policy(kobj); 851 struct freq_attr *fattr = to_attr(attr); 852 ssize_t ret; 853 854 if (!down_read_trylock(&cpufreq_rwsem)) 855 return -EINVAL; 856 857 down_read(&policy->rwsem); 858 859 if (fattr->show) 860 ret = fattr->show(policy, buf); 861 else 862 ret = -EIO; 863 864 up_read(&policy->rwsem); 865 up_read(&cpufreq_rwsem); 866 867 return ret; 868 } 869 870 static ssize_t store(struct kobject *kobj, struct attribute *attr, 871 const char *buf, size_t count) 872 { 873 struct cpufreq_policy *policy = to_policy(kobj); 874 struct freq_attr *fattr = to_attr(attr); 875 ssize_t ret = -EINVAL; 876 877 get_online_cpus(); 878 879 if (!cpu_online(policy->cpu)) 880 goto unlock; 881 882 if (!down_read_trylock(&cpufreq_rwsem)) 883 goto unlock; 884 885 down_write(&policy->rwsem); 886 887 /* Updating inactive policies is invalid, so avoid doing that. */ 888 if (unlikely(policy_is_inactive(policy))) { 889 ret = -EBUSY; 890 goto unlock_policy_rwsem; 891 } 892 893 if (fattr->store) 894 ret = fattr->store(policy, buf, count); 895 else 896 ret = -EIO; 897 898 unlock_policy_rwsem: 899 up_write(&policy->rwsem); 900 901 up_read(&cpufreq_rwsem); 902 unlock: 903 put_online_cpus(); 904 905 return ret; 906 } 907 908 static void cpufreq_sysfs_release(struct kobject *kobj) 909 { 910 struct cpufreq_policy *policy = to_policy(kobj); 911 pr_debug("last reference is dropped\n"); 912 complete(&policy->kobj_unregister); 913 } 914 915 static const struct sysfs_ops sysfs_ops = { 916 .show = show, 917 .store = store, 918 }; 919 920 static struct kobj_type ktype_cpufreq = { 921 .sysfs_ops = &sysfs_ops, 922 .default_attrs = default_attrs, 923 .release = cpufreq_sysfs_release, 924 }; 925 926 struct kobject *cpufreq_global_kobject; 927 EXPORT_SYMBOL(cpufreq_global_kobject); 928 929 static int cpufreq_global_kobject_usage; 930 931 int cpufreq_get_global_kobject(void) 932 { 933 if (!cpufreq_global_kobject_usage++) 934 return kobject_add(cpufreq_global_kobject, 935 &cpu_subsys.dev_root->kobj, "%s", "cpufreq"); 936 937 return 0; 938 } 939 EXPORT_SYMBOL(cpufreq_get_global_kobject); 940 941 void cpufreq_put_global_kobject(void) 942 { 943 if (!--cpufreq_global_kobject_usage) 944 kobject_del(cpufreq_global_kobject); 945 } 946 EXPORT_SYMBOL(cpufreq_put_global_kobject); 947 948 int cpufreq_sysfs_create_file(const struct attribute *attr) 949 { 950 int ret = cpufreq_get_global_kobject(); 951 952 if (!ret) { 953 ret = sysfs_create_file(cpufreq_global_kobject, attr); 954 if (ret) 955 cpufreq_put_global_kobject(); 956 } 957 958 return ret; 959 } 960 EXPORT_SYMBOL(cpufreq_sysfs_create_file); 961 962 void cpufreq_sysfs_remove_file(const struct attribute *attr) 963 { 964 sysfs_remove_file(cpufreq_global_kobject, attr); 965 cpufreq_put_global_kobject(); 966 } 967 EXPORT_SYMBOL(cpufreq_sysfs_remove_file); 968 969 static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu) 970 { 971 struct device *cpu_dev; 972 973 pr_debug("%s: Adding symlink for CPU: %u\n", __func__, cpu); 974 975 if (!policy) 976 return 0; 977 978 cpu_dev = get_cpu_device(cpu); 979 if (WARN_ON(!cpu_dev)) 980 return 0; 981 982 return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq"); 983 } 984 985 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu) 986 { 987 struct device *cpu_dev; 988 989 pr_debug("%s: Removing symlink for CPU: %u\n", __func__, cpu); 990 991 cpu_dev = get_cpu_device(cpu); 992 if (WARN_ON(!cpu_dev)) 993 return; 994 995 sysfs_remove_link(&cpu_dev->kobj, "cpufreq"); 996 } 997 998 /* Add/remove symlinks for all related CPUs */ 999 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy) 1000 { 1001 unsigned int j; 1002 int ret = 0; 1003 1004 /* Some related CPUs might not be present (physically hotplugged) */ 1005 for_each_cpu(j, policy->real_cpus) { 1006 if (j == policy->kobj_cpu) 1007 continue; 1008 1009 ret = add_cpu_dev_symlink(policy, j); 1010 if (ret) 1011 break; 1012 } 1013 1014 return ret; 1015 } 1016 1017 static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy) 1018 { 1019 unsigned int j; 1020 1021 /* Some related CPUs might not be present (physically hotplugged) */ 1022 for_each_cpu(j, policy->real_cpus) { 1023 if (j == policy->kobj_cpu) 1024 continue; 1025 1026 remove_cpu_dev_symlink(policy, j); 1027 } 1028 } 1029 1030 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy, 1031 struct device *dev) 1032 { 1033 struct freq_attr **drv_attr; 1034 int ret = 0; 1035 1036 /* set up files for this cpu device */ 1037 drv_attr = cpufreq_driver->attr; 1038 while (drv_attr && *drv_attr) { 1039 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr)); 1040 if (ret) 1041 return ret; 1042 drv_attr++; 1043 } 1044 if (cpufreq_driver->get) { 1045 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr); 1046 if (ret) 1047 return ret; 1048 } 1049 1050 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr); 1051 if (ret) 1052 return ret; 1053 1054 if (cpufreq_driver->bios_limit) { 1055 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr); 1056 if (ret) 1057 return ret; 1058 } 1059 1060 return cpufreq_add_dev_symlink(policy); 1061 } 1062 1063 static void cpufreq_init_policy(struct cpufreq_policy *policy) 1064 { 1065 struct cpufreq_governor *gov = NULL; 1066 struct cpufreq_policy new_policy; 1067 int ret = 0; 1068 1069 memcpy(&new_policy, policy, sizeof(*policy)); 1070 1071 /* Update governor of new_policy to the governor used before hotplug */ 1072 gov = find_governor(policy->last_governor); 1073 if (gov) 1074 pr_debug("Restoring governor %s for cpu %d\n", 1075 policy->governor->name, policy->cpu); 1076 else 1077 gov = CPUFREQ_DEFAULT_GOVERNOR; 1078 1079 new_policy.governor = gov; 1080 1081 /* Use the default policy if its valid. */ 1082 if (cpufreq_driver->setpolicy) 1083 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL); 1084 1085 /* set default policy */ 1086 ret = cpufreq_set_policy(policy, &new_policy); 1087 if (ret) { 1088 pr_debug("setting policy failed\n"); 1089 if (cpufreq_driver->exit) 1090 cpufreq_driver->exit(policy); 1091 } 1092 } 1093 1094 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, 1095 unsigned int cpu, struct device *dev) 1096 { 1097 int ret = 0; 1098 1099 /* Has this CPU been taken care of already? */ 1100 if (cpumask_test_cpu(cpu, policy->cpus)) 1101 return 0; 1102 1103 if (has_target()) { 1104 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); 1105 if (ret) { 1106 pr_err("%s: Failed to stop governor\n", __func__); 1107 return ret; 1108 } 1109 } 1110 1111 down_write(&policy->rwsem); 1112 cpumask_set_cpu(cpu, policy->cpus); 1113 up_write(&policy->rwsem); 1114 1115 if (has_target()) { 1116 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START); 1117 if (!ret) 1118 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); 1119 1120 if (ret) { 1121 pr_err("%s: Failed to start governor\n", __func__); 1122 return ret; 1123 } 1124 } 1125 1126 return 0; 1127 } 1128 1129 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu) 1130 { 1131 struct cpufreq_policy *policy; 1132 unsigned long flags; 1133 1134 read_lock_irqsave(&cpufreq_driver_lock, flags); 1135 policy = per_cpu(cpufreq_cpu_data, cpu); 1136 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1137 1138 if (likely(policy)) { 1139 /* Policy should be inactive here */ 1140 WARN_ON(!policy_is_inactive(policy)); 1141 1142 down_write(&policy->rwsem); 1143 policy->cpu = cpu; 1144 policy->governor = NULL; 1145 up_write(&policy->rwsem); 1146 } 1147 1148 return policy; 1149 } 1150 1151 static struct cpufreq_policy *cpufreq_policy_alloc(struct device *dev) 1152 { 1153 struct cpufreq_policy *policy; 1154 int ret; 1155 1156 policy = kzalloc(sizeof(*policy), GFP_KERNEL); 1157 if (!policy) 1158 return NULL; 1159 1160 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL)) 1161 goto err_free_policy; 1162 1163 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) 1164 goto err_free_cpumask; 1165 1166 if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL)) 1167 goto err_free_rcpumask; 1168 1169 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, &dev->kobj, 1170 "cpufreq"); 1171 if (ret) { 1172 pr_err("%s: failed to init policy->kobj: %d\n", __func__, ret); 1173 goto err_free_real_cpus; 1174 } 1175 1176 INIT_LIST_HEAD(&policy->policy_list); 1177 init_rwsem(&policy->rwsem); 1178 spin_lock_init(&policy->transition_lock); 1179 init_waitqueue_head(&policy->transition_wait); 1180 init_completion(&policy->kobj_unregister); 1181 INIT_WORK(&policy->update, handle_update); 1182 1183 policy->cpu = dev->id; 1184 1185 /* Set this once on allocation */ 1186 policy->kobj_cpu = dev->id; 1187 1188 return policy; 1189 1190 err_free_real_cpus: 1191 free_cpumask_var(policy->real_cpus); 1192 err_free_rcpumask: 1193 free_cpumask_var(policy->related_cpus); 1194 err_free_cpumask: 1195 free_cpumask_var(policy->cpus); 1196 err_free_policy: 1197 kfree(policy); 1198 1199 return NULL; 1200 } 1201 1202 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify) 1203 { 1204 struct kobject *kobj; 1205 struct completion *cmp; 1206 1207 if (notify) 1208 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1209 CPUFREQ_REMOVE_POLICY, policy); 1210 1211 down_write(&policy->rwsem); 1212 cpufreq_remove_dev_symlink(policy); 1213 kobj = &policy->kobj; 1214 cmp = &policy->kobj_unregister; 1215 up_write(&policy->rwsem); 1216 kobject_put(kobj); 1217 1218 /* 1219 * We need to make sure that the underlying kobj is 1220 * actually not referenced anymore by anybody before we 1221 * proceed with unloading. 1222 */ 1223 pr_debug("waiting for dropping of refcount\n"); 1224 wait_for_completion(cmp); 1225 pr_debug("wait complete\n"); 1226 } 1227 1228 static void cpufreq_policy_free(struct cpufreq_policy *policy, bool notify) 1229 { 1230 unsigned long flags; 1231 int cpu; 1232 1233 /* Remove policy from list */ 1234 write_lock_irqsave(&cpufreq_driver_lock, flags); 1235 list_del(&policy->policy_list); 1236 1237 for_each_cpu(cpu, policy->related_cpus) 1238 per_cpu(cpufreq_cpu_data, cpu) = NULL; 1239 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1240 1241 cpufreq_policy_put_kobj(policy, notify); 1242 free_cpumask_var(policy->real_cpus); 1243 free_cpumask_var(policy->related_cpus); 1244 free_cpumask_var(policy->cpus); 1245 kfree(policy); 1246 } 1247 1248 /** 1249 * cpufreq_add_dev - add a CPU device 1250 * 1251 * Adds the cpufreq interface for a CPU device. 1252 * 1253 * The Oracle says: try running cpufreq registration/unregistration concurrently 1254 * with with cpu hotplugging and all hell will break loose. Tried to clean this 1255 * mess up, but more thorough testing is needed. - Mathieu 1256 */ 1257 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) 1258 { 1259 unsigned int j, cpu = dev->id; 1260 int ret = -ENOMEM; 1261 struct cpufreq_policy *policy; 1262 unsigned long flags; 1263 bool recover_policy = !sif; 1264 1265 pr_debug("adding CPU %u\n", cpu); 1266 1267 if (cpu_is_offline(cpu)) { 1268 /* 1269 * Only possible if we are here from the subsys_interface add 1270 * callback. A hotplug notifier will follow and we will handle 1271 * it as CPU online then. For now, just create the sysfs link, 1272 * unless there is no policy or the link is already present. 1273 */ 1274 policy = per_cpu(cpufreq_cpu_data, cpu); 1275 return policy && !cpumask_test_and_set_cpu(cpu, policy->real_cpus) 1276 ? add_cpu_dev_symlink(policy, cpu) : 0; 1277 } 1278 1279 if (!down_read_trylock(&cpufreq_rwsem)) 1280 return 0; 1281 1282 /* Check if this CPU already has a policy to manage it */ 1283 policy = per_cpu(cpufreq_cpu_data, cpu); 1284 if (policy && !policy_is_inactive(policy)) { 1285 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); 1286 ret = cpufreq_add_policy_cpu(policy, cpu, dev); 1287 up_read(&cpufreq_rwsem); 1288 return ret; 1289 } 1290 1291 /* 1292 * Restore the saved policy when doing light-weight init and fall back 1293 * to the full init if that fails. 1294 */ 1295 policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL; 1296 if (!policy) { 1297 recover_policy = false; 1298 policy = cpufreq_policy_alloc(dev); 1299 if (!policy) 1300 goto nomem_out; 1301 } 1302 1303 cpumask_copy(policy->cpus, cpumask_of(cpu)); 1304 1305 /* call driver. From then on the cpufreq must be able 1306 * to accept all calls to ->verify and ->setpolicy for this CPU 1307 */ 1308 ret = cpufreq_driver->init(policy); 1309 if (ret) { 1310 pr_debug("initialization failed\n"); 1311 goto err_set_policy_cpu; 1312 } 1313 1314 down_write(&policy->rwsem); 1315 1316 /* related cpus should atleast have policy->cpus */ 1317 cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus); 1318 1319 /* Remember which CPUs have been present at the policy creation time. */ 1320 if (!recover_policy) 1321 cpumask_and(policy->real_cpus, policy->cpus, cpu_present_mask); 1322 1323 /* 1324 * affected cpus must always be the one, which are online. We aren't 1325 * managing offline cpus here. 1326 */ 1327 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); 1328 1329 if (!recover_policy) { 1330 policy->user_policy.min = policy->min; 1331 policy->user_policy.max = policy->max; 1332 1333 write_lock_irqsave(&cpufreq_driver_lock, flags); 1334 for_each_cpu(j, policy->related_cpus) 1335 per_cpu(cpufreq_cpu_data, j) = policy; 1336 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1337 } 1338 1339 if (cpufreq_driver->get && !cpufreq_driver->setpolicy) { 1340 policy->cur = cpufreq_driver->get(policy->cpu); 1341 if (!policy->cur) { 1342 pr_err("%s: ->get() failed\n", __func__); 1343 goto err_get_freq; 1344 } 1345 } 1346 1347 /* 1348 * Sometimes boot loaders set CPU frequency to a value outside of 1349 * frequency table present with cpufreq core. In such cases CPU might be 1350 * unstable if it has to run on that frequency for long duration of time 1351 * and so its better to set it to a frequency which is specified in 1352 * freq-table. This also makes cpufreq stats inconsistent as 1353 * cpufreq-stats would fail to register because current frequency of CPU 1354 * isn't found in freq-table. 1355 * 1356 * Because we don't want this change to effect boot process badly, we go 1357 * for the next freq which is >= policy->cur ('cur' must be set by now, 1358 * otherwise we will end up setting freq to lowest of the table as 'cur' 1359 * is initialized to zero). 1360 * 1361 * We are passing target-freq as "policy->cur - 1" otherwise 1362 * __cpufreq_driver_target() would simply fail, as policy->cur will be 1363 * equal to target-freq. 1364 */ 1365 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK) 1366 && has_target()) { 1367 /* Are we running at unknown frequency ? */ 1368 ret = cpufreq_frequency_table_get_index(policy, policy->cur); 1369 if (ret == -EINVAL) { 1370 /* Warn user and fix it */ 1371 pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n", 1372 __func__, policy->cpu, policy->cur); 1373 ret = __cpufreq_driver_target(policy, policy->cur - 1, 1374 CPUFREQ_RELATION_L); 1375 1376 /* 1377 * Reaching here after boot in a few seconds may not 1378 * mean that system will remain stable at "unknown" 1379 * frequency for longer duration. Hence, a BUG_ON(). 1380 */ 1381 BUG_ON(ret); 1382 pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n", 1383 __func__, policy->cpu, policy->cur); 1384 } 1385 } 1386 1387 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1388 CPUFREQ_START, policy); 1389 1390 if (!recover_policy) { 1391 ret = cpufreq_add_dev_interface(policy, dev); 1392 if (ret) 1393 goto err_out_unregister; 1394 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 1395 CPUFREQ_CREATE_POLICY, policy); 1396 1397 write_lock_irqsave(&cpufreq_driver_lock, flags); 1398 list_add(&policy->policy_list, &cpufreq_policy_list); 1399 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1400 } 1401 1402 cpufreq_init_policy(policy); 1403 1404 if (!recover_policy) { 1405 policy->user_policy.policy = policy->policy; 1406 policy->user_policy.governor = policy->governor; 1407 } 1408 up_write(&policy->rwsem); 1409 1410 kobject_uevent(&policy->kobj, KOBJ_ADD); 1411 1412 up_read(&cpufreq_rwsem); 1413 1414 /* Callback for handling stuff after policy is ready */ 1415 if (cpufreq_driver->ready) 1416 cpufreq_driver->ready(policy); 1417 1418 pr_debug("initialization complete\n"); 1419 1420 return 0; 1421 1422 err_out_unregister: 1423 err_get_freq: 1424 up_write(&policy->rwsem); 1425 1426 if (cpufreq_driver->exit) 1427 cpufreq_driver->exit(policy); 1428 err_set_policy_cpu: 1429 cpufreq_policy_free(policy, recover_policy); 1430 nomem_out: 1431 up_read(&cpufreq_rwsem); 1432 1433 return ret; 1434 } 1435 1436 static int __cpufreq_remove_dev_prepare(struct device *dev) 1437 { 1438 unsigned int cpu = dev->id; 1439 int ret = 0; 1440 struct cpufreq_policy *policy; 1441 1442 pr_debug("%s: unregistering CPU %u\n", __func__, cpu); 1443 1444 policy = cpufreq_cpu_get_raw(cpu); 1445 if (!policy) { 1446 pr_debug("%s: No cpu_data found\n", __func__); 1447 return -EINVAL; 1448 } 1449 1450 if (has_target()) { 1451 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); 1452 if (ret) 1453 pr_err("%s: Failed to stop governor\n", __func__); 1454 } 1455 1456 down_write(&policy->rwsem); 1457 cpumask_clear_cpu(cpu, policy->cpus); 1458 1459 if (policy_is_inactive(policy)) { 1460 if (has_target()) 1461 strncpy(policy->last_governor, policy->governor->name, 1462 CPUFREQ_NAME_LEN); 1463 } else if (cpu == policy->cpu) { 1464 /* Nominate new CPU */ 1465 policy->cpu = cpumask_any(policy->cpus); 1466 } 1467 up_write(&policy->rwsem); 1468 1469 /* Start governor again for active policy */ 1470 if (!policy_is_inactive(policy)) { 1471 if (has_target()) { 1472 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START); 1473 if (!ret) 1474 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); 1475 1476 if (ret) 1477 pr_err("%s: Failed to start governor\n", __func__); 1478 } 1479 } else if (cpufreq_driver->stop_cpu) { 1480 cpufreq_driver->stop_cpu(policy); 1481 } 1482 1483 return ret; 1484 } 1485 1486 static int __cpufreq_remove_dev_finish(struct device *dev) 1487 { 1488 unsigned int cpu = dev->id; 1489 int ret; 1490 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 1491 1492 if (!policy) { 1493 pr_debug("%s: No cpu_data found\n", __func__); 1494 return -EINVAL; 1495 } 1496 1497 /* Only proceed for inactive policies */ 1498 if (!policy_is_inactive(policy)) 1499 return 0; 1500 1501 /* If cpu is last user of policy, free policy */ 1502 if (has_target()) { 1503 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); 1504 if (ret) 1505 pr_err("%s: Failed to exit governor\n", __func__); 1506 } 1507 1508 /* 1509 * Perform the ->exit() even during light-weight tear-down, 1510 * since this is a core component, and is essential for the 1511 * subsequent light-weight ->init() to succeed. 1512 */ 1513 if (cpufreq_driver->exit) 1514 cpufreq_driver->exit(policy); 1515 1516 return 0; 1517 } 1518 1519 /** 1520 * cpufreq_remove_dev - remove a CPU device 1521 * 1522 * Removes the cpufreq interface for a CPU device. 1523 */ 1524 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) 1525 { 1526 unsigned int cpu = dev->id; 1527 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 1528 1529 if (!policy) 1530 return 0; 1531 1532 if (cpu_online(cpu)) { 1533 __cpufreq_remove_dev_prepare(dev); 1534 __cpufreq_remove_dev_finish(dev); 1535 } 1536 1537 cpumask_clear_cpu(cpu, policy->real_cpus); 1538 1539 if (cpumask_empty(policy->real_cpus)) { 1540 cpufreq_policy_free(policy, true); 1541 return 0; 1542 } 1543 1544 if (cpu != policy->kobj_cpu) { 1545 remove_cpu_dev_symlink(policy, cpu); 1546 } else { 1547 /* 1548 * The CPU owning the policy object is going away. Move it to 1549 * another suitable CPU. 1550 */ 1551 unsigned int new_cpu = cpumask_first(policy->real_cpus); 1552 struct device *new_dev = get_cpu_device(new_cpu); 1553 1554 dev_dbg(dev, "%s: Moving policy object to CPU%u\n", __func__, new_cpu); 1555 1556 sysfs_remove_link(&new_dev->kobj, "cpufreq"); 1557 policy->kobj_cpu = new_cpu; 1558 WARN_ON(kobject_move(&policy->kobj, &new_dev->kobj)); 1559 } 1560 1561 return 0; 1562 } 1563 1564 static void handle_update(struct work_struct *work) 1565 { 1566 struct cpufreq_policy *policy = 1567 container_of(work, struct cpufreq_policy, update); 1568 unsigned int cpu = policy->cpu; 1569 pr_debug("handle_update for cpu %u called\n", cpu); 1570 cpufreq_update_policy(cpu); 1571 } 1572 1573 /** 1574 * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're 1575 * in deep trouble. 1576 * @policy: policy managing CPUs 1577 * @new_freq: CPU frequency the CPU actually runs at 1578 * 1579 * We adjust to current frequency first, and need to clean up later. 1580 * So either call to cpufreq_update_policy() or schedule handle_update()). 1581 */ 1582 static void cpufreq_out_of_sync(struct cpufreq_policy *policy, 1583 unsigned int new_freq) 1584 { 1585 struct cpufreq_freqs freqs; 1586 1587 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n", 1588 policy->cur, new_freq); 1589 1590 freqs.old = policy->cur; 1591 freqs.new = new_freq; 1592 1593 cpufreq_freq_transition_begin(policy, &freqs); 1594 cpufreq_freq_transition_end(policy, &freqs, 0); 1595 } 1596 1597 /** 1598 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur 1599 * @cpu: CPU number 1600 * 1601 * This is the last known freq, without actually getting it from the driver. 1602 * Return value will be same as what is shown in scaling_cur_freq in sysfs. 1603 */ 1604 unsigned int cpufreq_quick_get(unsigned int cpu) 1605 { 1606 struct cpufreq_policy *policy; 1607 unsigned int ret_freq = 0; 1608 1609 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) 1610 return cpufreq_driver->get(cpu); 1611 1612 policy = cpufreq_cpu_get(cpu); 1613 if (policy) { 1614 ret_freq = policy->cur; 1615 cpufreq_cpu_put(policy); 1616 } 1617 1618 return ret_freq; 1619 } 1620 EXPORT_SYMBOL(cpufreq_quick_get); 1621 1622 /** 1623 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU 1624 * @cpu: CPU number 1625 * 1626 * Just return the max possible frequency for a given CPU. 1627 */ 1628 unsigned int cpufreq_quick_get_max(unsigned int cpu) 1629 { 1630 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1631 unsigned int ret_freq = 0; 1632 1633 if (policy) { 1634 ret_freq = policy->max; 1635 cpufreq_cpu_put(policy); 1636 } 1637 1638 return ret_freq; 1639 } 1640 EXPORT_SYMBOL(cpufreq_quick_get_max); 1641 1642 static unsigned int __cpufreq_get(struct cpufreq_policy *policy) 1643 { 1644 unsigned int ret_freq = 0; 1645 1646 if (!cpufreq_driver->get) 1647 return ret_freq; 1648 1649 ret_freq = cpufreq_driver->get(policy->cpu); 1650 1651 /* Updating inactive policies is invalid, so avoid doing that. */ 1652 if (unlikely(policy_is_inactive(policy))) 1653 return ret_freq; 1654 1655 if (ret_freq && policy->cur && 1656 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) { 1657 /* verify no discrepancy between actual and 1658 saved value exists */ 1659 if (unlikely(ret_freq != policy->cur)) { 1660 cpufreq_out_of_sync(policy, ret_freq); 1661 schedule_work(&policy->update); 1662 } 1663 } 1664 1665 return ret_freq; 1666 } 1667 1668 /** 1669 * cpufreq_get - get the current CPU frequency (in kHz) 1670 * @cpu: CPU number 1671 * 1672 * Get the CPU current (static) CPU frequency 1673 */ 1674 unsigned int cpufreq_get(unsigned int cpu) 1675 { 1676 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1677 unsigned int ret_freq = 0; 1678 1679 if (policy) { 1680 down_read(&policy->rwsem); 1681 ret_freq = __cpufreq_get(policy); 1682 up_read(&policy->rwsem); 1683 1684 cpufreq_cpu_put(policy); 1685 } 1686 1687 return ret_freq; 1688 } 1689 EXPORT_SYMBOL(cpufreq_get); 1690 1691 static struct subsys_interface cpufreq_interface = { 1692 .name = "cpufreq", 1693 .subsys = &cpu_subsys, 1694 .add_dev = cpufreq_add_dev, 1695 .remove_dev = cpufreq_remove_dev, 1696 }; 1697 1698 /* 1699 * In case platform wants some specific frequency to be configured 1700 * during suspend.. 1701 */ 1702 int cpufreq_generic_suspend(struct cpufreq_policy *policy) 1703 { 1704 int ret; 1705 1706 if (!policy->suspend_freq) { 1707 pr_err("%s: suspend_freq can't be zero\n", __func__); 1708 return -EINVAL; 1709 } 1710 1711 pr_debug("%s: Setting suspend-freq: %u\n", __func__, 1712 policy->suspend_freq); 1713 1714 ret = __cpufreq_driver_target(policy, policy->suspend_freq, 1715 CPUFREQ_RELATION_H); 1716 if (ret) 1717 pr_err("%s: unable to set suspend-freq: %u. err: %d\n", 1718 __func__, policy->suspend_freq, ret); 1719 1720 return ret; 1721 } 1722 EXPORT_SYMBOL(cpufreq_generic_suspend); 1723 1724 /** 1725 * cpufreq_suspend() - Suspend CPUFreq governors 1726 * 1727 * Called during system wide Suspend/Hibernate cycles for suspending governors 1728 * as some platforms can't change frequency after this point in suspend cycle. 1729 * Because some of the devices (like: i2c, regulators, etc) they use for 1730 * changing frequency are suspended quickly after this point. 1731 */ 1732 void cpufreq_suspend(void) 1733 { 1734 struct cpufreq_policy *policy; 1735 1736 if (!cpufreq_driver) 1737 return; 1738 1739 if (!has_target()) 1740 goto suspend; 1741 1742 pr_debug("%s: Suspending Governors\n", __func__); 1743 1744 for_each_active_policy(policy) { 1745 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP)) 1746 pr_err("%s: Failed to stop governor for policy: %p\n", 1747 __func__, policy); 1748 else if (cpufreq_driver->suspend 1749 && cpufreq_driver->suspend(policy)) 1750 pr_err("%s: Failed to suspend driver: %p\n", __func__, 1751 policy); 1752 } 1753 1754 suspend: 1755 cpufreq_suspended = true; 1756 } 1757 1758 /** 1759 * cpufreq_resume() - Resume CPUFreq governors 1760 * 1761 * Called during system wide Suspend/Hibernate cycle for resuming governors that 1762 * are suspended with cpufreq_suspend(). 1763 */ 1764 void cpufreq_resume(void) 1765 { 1766 struct cpufreq_policy *policy; 1767 1768 if (!cpufreq_driver) 1769 return; 1770 1771 cpufreq_suspended = false; 1772 1773 if (!has_target()) 1774 return; 1775 1776 pr_debug("%s: Resuming Governors\n", __func__); 1777 1778 for_each_active_policy(policy) { 1779 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) 1780 pr_err("%s: Failed to resume driver: %p\n", __func__, 1781 policy); 1782 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START) 1783 || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS)) 1784 pr_err("%s: Failed to start governor for policy: %p\n", 1785 __func__, policy); 1786 } 1787 1788 /* 1789 * schedule call cpufreq_update_policy() for first-online CPU, as that 1790 * wouldn't be hotplugged-out on suspend. It will verify that the 1791 * current freq is in sync with what we believe it to be. 1792 */ 1793 policy = cpufreq_cpu_get_raw(cpumask_first(cpu_online_mask)); 1794 if (WARN_ON(!policy)) 1795 return; 1796 1797 schedule_work(&policy->update); 1798 } 1799 1800 /** 1801 * cpufreq_get_current_driver - return current driver's name 1802 * 1803 * Return the name string of the currently loaded cpufreq driver 1804 * or NULL, if none. 1805 */ 1806 const char *cpufreq_get_current_driver(void) 1807 { 1808 if (cpufreq_driver) 1809 return cpufreq_driver->name; 1810 1811 return NULL; 1812 } 1813 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver); 1814 1815 /** 1816 * cpufreq_get_driver_data - return current driver data 1817 * 1818 * Return the private data of the currently loaded cpufreq 1819 * driver, or NULL if no cpufreq driver is loaded. 1820 */ 1821 void *cpufreq_get_driver_data(void) 1822 { 1823 if (cpufreq_driver) 1824 return cpufreq_driver->driver_data; 1825 1826 return NULL; 1827 } 1828 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data); 1829 1830 /********************************************************************* 1831 * NOTIFIER LISTS INTERFACE * 1832 *********************************************************************/ 1833 1834 /** 1835 * cpufreq_register_notifier - register a driver with cpufreq 1836 * @nb: notifier function to register 1837 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER 1838 * 1839 * Add a driver to one of two lists: either a list of drivers that 1840 * are notified about clock rate changes (once before and once after 1841 * the transition), or a list of drivers that are notified about 1842 * changes in cpufreq policy. 1843 * 1844 * This function may sleep, and has the same return conditions as 1845 * blocking_notifier_chain_register. 1846 */ 1847 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list) 1848 { 1849 int ret; 1850 1851 if (cpufreq_disabled()) 1852 return -EINVAL; 1853 1854 WARN_ON(!init_cpufreq_transition_notifier_list_called); 1855 1856 switch (list) { 1857 case CPUFREQ_TRANSITION_NOTIFIER: 1858 ret = srcu_notifier_chain_register( 1859 &cpufreq_transition_notifier_list, nb); 1860 break; 1861 case CPUFREQ_POLICY_NOTIFIER: 1862 ret = blocking_notifier_chain_register( 1863 &cpufreq_policy_notifier_list, nb); 1864 break; 1865 default: 1866 ret = -EINVAL; 1867 } 1868 1869 return ret; 1870 } 1871 EXPORT_SYMBOL(cpufreq_register_notifier); 1872 1873 /** 1874 * cpufreq_unregister_notifier - unregister a driver with cpufreq 1875 * @nb: notifier block to be unregistered 1876 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER 1877 * 1878 * Remove a driver from the CPU frequency notifier list. 1879 * 1880 * This function may sleep, and has the same return conditions as 1881 * blocking_notifier_chain_unregister. 1882 */ 1883 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list) 1884 { 1885 int ret; 1886 1887 if (cpufreq_disabled()) 1888 return -EINVAL; 1889 1890 switch (list) { 1891 case CPUFREQ_TRANSITION_NOTIFIER: 1892 ret = srcu_notifier_chain_unregister( 1893 &cpufreq_transition_notifier_list, nb); 1894 break; 1895 case CPUFREQ_POLICY_NOTIFIER: 1896 ret = blocking_notifier_chain_unregister( 1897 &cpufreq_policy_notifier_list, nb); 1898 break; 1899 default: 1900 ret = -EINVAL; 1901 } 1902 1903 return ret; 1904 } 1905 EXPORT_SYMBOL(cpufreq_unregister_notifier); 1906 1907 1908 /********************************************************************* 1909 * GOVERNORS * 1910 *********************************************************************/ 1911 1912 /* Must set freqs->new to intermediate frequency */ 1913 static int __target_intermediate(struct cpufreq_policy *policy, 1914 struct cpufreq_freqs *freqs, int index) 1915 { 1916 int ret; 1917 1918 freqs->new = cpufreq_driver->get_intermediate(policy, index); 1919 1920 /* We don't need to switch to intermediate freq */ 1921 if (!freqs->new) 1922 return 0; 1923 1924 pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n", 1925 __func__, policy->cpu, freqs->old, freqs->new); 1926 1927 cpufreq_freq_transition_begin(policy, freqs); 1928 ret = cpufreq_driver->target_intermediate(policy, index); 1929 cpufreq_freq_transition_end(policy, freqs, ret); 1930 1931 if (ret) 1932 pr_err("%s: Failed to change to intermediate frequency: %d\n", 1933 __func__, ret); 1934 1935 return ret; 1936 } 1937 1938 static int __target_index(struct cpufreq_policy *policy, 1939 struct cpufreq_frequency_table *freq_table, int index) 1940 { 1941 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0}; 1942 unsigned int intermediate_freq = 0; 1943 int retval = -EINVAL; 1944 bool notify; 1945 1946 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION); 1947 if (notify) { 1948 /* Handle switching to intermediate frequency */ 1949 if (cpufreq_driver->get_intermediate) { 1950 retval = __target_intermediate(policy, &freqs, index); 1951 if (retval) 1952 return retval; 1953 1954 intermediate_freq = freqs.new; 1955 /* Set old freq to intermediate */ 1956 if (intermediate_freq) 1957 freqs.old = freqs.new; 1958 } 1959 1960 freqs.new = freq_table[index].frequency; 1961 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n", 1962 __func__, policy->cpu, freqs.old, freqs.new); 1963 1964 cpufreq_freq_transition_begin(policy, &freqs); 1965 } 1966 1967 retval = cpufreq_driver->target_index(policy, index); 1968 if (retval) 1969 pr_err("%s: Failed to change cpu frequency: %d\n", __func__, 1970 retval); 1971 1972 if (notify) { 1973 cpufreq_freq_transition_end(policy, &freqs, retval); 1974 1975 /* 1976 * Failed after setting to intermediate freq? Driver should have 1977 * reverted back to initial frequency and so should we. Check 1978 * here for intermediate_freq instead of get_intermediate, in 1979 * case we haven't switched to intermediate freq at all. 1980 */ 1981 if (unlikely(retval && intermediate_freq)) { 1982 freqs.old = intermediate_freq; 1983 freqs.new = policy->restore_freq; 1984 cpufreq_freq_transition_begin(policy, &freqs); 1985 cpufreq_freq_transition_end(policy, &freqs, 0); 1986 } 1987 } 1988 1989 return retval; 1990 } 1991 1992 int __cpufreq_driver_target(struct cpufreq_policy *policy, 1993 unsigned int target_freq, 1994 unsigned int relation) 1995 { 1996 unsigned int old_target_freq = target_freq; 1997 int retval = -EINVAL; 1998 1999 if (cpufreq_disabled()) 2000 return -ENODEV; 2001 2002 /* Make sure that target_freq is within supported range */ 2003 if (target_freq > policy->max) 2004 target_freq = policy->max; 2005 if (target_freq < policy->min) 2006 target_freq = policy->min; 2007 2008 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n", 2009 policy->cpu, target_freq, relation, old_target_freq); 2010 2011 /* 2012 * This might look like a redundant call as we are checking it again 2013 * after finding index. But it is left intentionally for cases where 2014 * exactly same freq is called again and so we can save on few function 2015 * calls. 2016 */ 2017 if (target_freq == policy->cur) 2018 return 0; 2019 2020 /* Save last value to restore later on errors */ 2021 policy->restore_freq = policy->cur; 2022 2023 if (cpufreq_driver->target) 2024 retval = cpufreq_driver->target(policy, target_freq, relation); 2025 else if (cpufreq_driver->target_index) { 2026 struct cpufreq_frequency_table *freq_table; 2027 int index; 2028 2029 freq_table = cpufreq_frequency_get_table(policy->cpu); 2030 if (unlikely(!freq_table)) { 2031 pr_err("%s: Unable to find freq_table\n", __func__); 2032 goto out; 2033 } 2034 2035 retval = cpufreq_frequency_table_target(policy, freq_table, 2036 target_freq, relation, &index); 2037 if (unlikely(retval)) { 2038 pr_err("%s: Unable to find matching freq\n", __func__); 2039 goto out; 2040 } 2041 2042 if (freq_table[index].frequency == policy->cur) { 2043 retval = 0; 2044 goto out; 2045 } 2046 2047 retval = __target_index(policy, freq_table, index); 2048 } 2049 2050 out: 2051 return retval; 2052 } 2053 EXPORT_SYMBOL_GPL(__cpufreq_driver_target); 2054 2055 int cpufreq_driver_target(struct cpufreq_policy *policy, 2056 unsigned int target_freq, 2057 unsigned int relation) 2058 { 2059 int ret = -EINVAL; 2060 2061 down_write(&policy->rwsem); 2062 2063 ret = __cpufreq_driver_target(policy, target_freq, relation); 2064 2065 up_write(&policy->rwsem); 2066 2067 return ret; 2068 } 2069 EXPORT_SYMBOL_GPL(cpufreq_driver_target); 2070 2071 static int __cpufreq_governor(struct cpufreq_policy *policy, 2072 unsigned int event) 2073 { 2074 int ret; 2075 2076 /* Only must be defined when default governor is known to have latency 2077 restrictions, like e.g. conservative or ondemand. 2078 That this is the case is already ensured in Kconfig 2079 */ 2080 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE 2081 struct cpufreq_governor *gov = &cpufreq_gov_performance; 2082 #else 2083 struct cpufreq_governor *gov = NULL; 2084 #endif 2085 2086 /* Don't start any governor operations if we are entering suspend */ 2087 if (cpufreq_suspended) 2088 return 0; 2089 /* 2090 * Governor might not be initiated here if ACPI _PPC changed 2091 * notification happened, so check it. 2092 */ 2093 if (!policy->governor) 2094 return -EINVAL; 2095 2096 if (policy->governor->max_transition_latency && 2097 policy->cpuinfo.transition_latency > 2098 policy->governor->max_transition_latency) { 2099 if (!gov) 2100 return -EINVAL; 2101 else { 2102 pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n", 2103 policy->governor->name, gov->name); 2104 policy->governor = gov; 2105 } 2106 } 2107 2108 if (event == CPUFREQ_GOV_POLICY_INIT) 2109 if (!try_module_get(policy->governor->owner)) 2110 return -EINVAL; 2111 2112 pr_debug("__cpufreq_governor for CPU %u, event %u\n", 2113 policy->cpu, event); 2114 2115 mutex_lock(&cpufreq_governor_lock); 2116 if ((policy->governor_enabled && event == CPUFREQ_GOV_START) 2117 || (!policy->governor_enabled 2118 && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) { 2119 mutex_unlock(&cpufreq_governor_lock); 2120 return -EBUSY; 2121 } 2122 2123 if (event == CPUFREQ_GOV_STOP) 2124 policy->governor_enabled = false; 2125 else if (event == CPUFREQ_GOV_START) 2126 policy->governor_enabled = true; 2127 2128 mutex_unlock(&cpufreq_governor_lock); 2129 2130 ret = policy->governor->governor(policy, event); 2131 2132 if (!ret) { 2133 if (event == CPUFREQ_GOV_POLICY_INIT) 2134 policy->governor->initialized++; 2135 else if (event == CPUFREQ_GOV_POLICY_EXIT) 2136 policy->governor->initialized--; 2137 } else { 2138 /* Restore original values */ 2139 mutex_lock(&cpufreq_governor_lock); 2140 if (event == CPUFREQ_GOV_STOP) 2141 policy->governor_enabled = true; 2142 else if (event == CPUFREQ_GOV_START) 2143 policy->governor_enabled = false; 2144 mutex_unlock(&cpufreq_governor_lock); 2145 } 2146 2147 if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) || 2148 ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret)) 2149 module_put(policy->governor->owner); 2150 2151 return ret; 2152 } 2153 2154 int cpufreq_register_governor(struct cpufreq_governor *governor) 2155 { 2156 int err; 2157 2158 if (!governor) 2159 return -EINVAL; 2160 2161 if (cpufreq_disabled()) 2162 return -ENODEV; 2163 2164 mutex_lock(&cpufreq_governor_mutex); 2165 2166 governor->initialized = 0; 2167 err = -EBUSY; 2168 if (!find_governor(governor->name)) { 2169 err = 0; 2170 list_add(&governor->governor_list, &cpufreq_governor_list); 2171 } 2172 2173 mutex_unlock(&cpufreq_governor_mutex); 2174 return err; 2175 } 2176 EXPORT_SYMBOL_GPL(cpufreq_register_governor); 2177 2178 void cpufreq_unregister_governor(struct cpufreq_governor *governor) 2179 { 2180 struct cpufreq_policy *policy; 2181 unsigned long flags; 2182 2183 if (!governor) 2184 return; 2185 2186 if (cpufreq_disabled()) 2187 return; 2188 2189 /* clear last_governor for all inactive policies */ 2190 read_lock_irqsave(&cpufreq_driver_lock, flags); 2191 for_each_inactive_policy(policy) { 2192 if (!strcmp(policy->last_governor, governor->name)) { 2193 policy->governor = NULL; 2194 strcpy(policy->last_governor, "\0"); 2195 } 2196 } 2197 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 2198 2199 mutex_lock(&cpufreq_governor_mutex); 2200 list_del(&governor->governor_list); 2201 mutex_unlock(&cpufreq_governor_mutex); 2202 return; 2203 } 2204 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor); 2205 2206 2207 /********************************************************************* 2208 * POLICY INTERFACE * 2209 *********************************************************************/ 2210 2211 /** 2212 * cpufreq_get_policy - get the current cpufreq_policy 2213 * @policy: struct cpufreq_policy into which the current cpufreq_policy 2214 * is written 2215 * 2216 * Reads the current cpufreq policy. 2217 */ 2218 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu) 2219 { 2220 struct cpufreq_policy *cpu_policy; 2221 if (!policy) 2222 return -EINVAL; 2223 2224 cpu_policy = cpufreq_cpu_get(cpu); 2225 if (!cpu_policy) 2226 return -EINVAL; 2227 2228 memcpy(policy, cpu_policy, sizeof(*policy)); 2229 2230 cpufreq_cpu_put(cpu_policy); 2231 return 0; 2232 } 2233 EXPORT_SYMBOL(cpufreq_get_policy); 2234 2235 /* 2236 * policy : current policy. 2237 * new_policy: policy to be set. 2238 */ 2239 static int cpufreq_set_policy(struct cpufreq_policy *policy, 2240 struct cpufreq_policy *new_policy) 2241 { 2242 struct cpufreq_governor *old_gov; 2243 int ret; 2244 2245 pr_debug("setting new policy for CPU %u: %u - %u kHz\n", 2246 new_policy->cpu, new_policy->min, new_policy->max); 2247 2248 memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo)); 2249 2250 if (new_policy->min > policy->max || new_policy->max < policy->min) 2251 return -EINVAL; 2252 2253 /* verify the cpu speed can be set within this limit */ 2254 ret = cpufreq_driver->verify(new_policy); 2255 if (ret) 2256 return ret; 2257 2258 /* adjust if necessary - all reasons */ 2259 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 2260 CPUFREQ_ADJUST, new_policy); 2261 2262 /* adjust if necessary - hardware incompatibility*/ 2263 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 2264 CPUFREQ_INCOMPATIBLE, new_policy); 2265 2266 /* 2267 * verify the cpu speed can be set within this limit, which might be 2268 * different to the first one 2269 */ 2270 ret = cpufreq_driver->verify(new_policy); 2271 if (ret) 2272 return ret; 2273 2274 /* notification of the new policy */ 2275 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 2276 CPUFREQ_NOTIFY, new_policy); 2277 2278 policy->min = new_policy->min; 2279 policy->max = new_policy->max; 2280 2281 pr_debug("new min and max freqs are %u - %u kHz\n", 2282 policy->min, policy->max); 2283 2284 if (cpufreq_driver->setpolicy) { 2285 policy->policy = new_policy->policy; 2286 pr_debug("setting range\n"); 2287 return cpufreq_driver->setpolicy(new_policy); 2288 } 2289 2290 if (new_policy->governor == policy->governor) 2291 goto out; 2292 2293 pr_debug("governor switch\n"); 2294 2295 /* save old, working values */ 2296 old_gov = policy->governor; 2297 /* end old governor */ 2298 if (old_gov) { 2299 __cpufreq_governor(policy, CPUFREQ_GOV_STOP); 2300 up_write(&policy->rwsem); 2301 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); 2302 down_write(&policy->rwsem); 2303 } 2304 2305 /* start new governor */ 2306 policy->governor = new_policy->governor; 2307 if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) { 2308 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START)) 2309 goto out; 2310 2311 up_write(&policy->rwsem); 2312 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); 2313 down_write(&policy->rwsem); 2314 } 2315 2316 /* new governor failed, so re-start old one */ 2317 pr_debug("starting governor %s failed\n", policy->governor->name); 2318 if (old_gov) { 2319 policy->governor = old_gov; 2320 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT); 2321 __cpufreq_governor(policy, CPUFREQ_GOV_START); 2322 } 2323 2324 return -EINVAL; 2325 2326 out: 2327 pr_debug("governor: change or update limits\n"); 2328 return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); 2329 } 2330 2331 /** 2332 * cpufreq_update_policy - re-evaluate an existing cpufreq policy 2333 * @cpu: CPU which shall be re-evaluated 2334 * 2335 * Useful for policy notifiers which have different necessities 2336 * at different times. 2337 */ 2338 int cpufreq_update_policy(unsigned int cpu) 2339 { 2340 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 2341 struct cpufreq_policy new_policy; 2342 int ret; 2343 2344 if (!policy) 2345 return -ENODEV; 2346 2347 down_write(&policy->rwsem); 2348 2349 pr_debug("updating policy for CPU %u\n", cpu); 2350 memcpy(&new_policy, policy, sizeof(*policy)); 2351 new_policy.min = policy->user_policy.min; 2352 new_policy.max = policy->user_policy.max; 2353 new_policy.policy = policy->user_policy.policy; 2354 new_policy.governor = policy->user_policy.governor; 2355 2356 /* 2357 * BIOS might change freq behind our back 2358 * -> ask driver for current freq and notify governors about a change 2359 */ 2360 if (cpufreq_driver->get && !cpufreq_driver->setpolicy) { 2361 new_policy.cur = cpufreq_driver->get(cpu); 2362 if (WARN_ON(!new_policy.cur)) { 2363 ret = -EIO; 2364 goto unlock; 2365 } 2366 2367 if (!policy->cur) { 2368 pr_debug("Driver did not initialize current freq\n"); 2369 policy->cur = new_policy.cur; 2370 } else { 2371 if (policy->cur != new_policy.cur && has_target()) 2372 cpufreq_out_of_sync(policy, new_policy.cur); 2373 } 2374 } 2375 2376 ret = cpufreq_set_policy(policy, &new_policy); 2377 2378 unlock: 2379 up_write(&policy->rwsem); 2380 2381 cpufreq_cpu_put(policy); 2382 return ret; 2383 } 2384 EXPORT_SYMBOL(cpufreq_update_policy); 2385 2386 static int cpufreq_cpu_callback(struct notifier_block *nfb, 2387 unsigned long action, void *hcpu) 2388 { 2389 unsigned int cpu = (unsigned long)hcpu; 2390 struct device *dev; 2391 2392 dev = get_cpu_device(cpu); 2393 if (dev) { 2394 switch (action & ~CPU_TASKS_FROZEN) { 2395 case CPU_ONLINE: 2396 cpufreq_add_dev(dev, NULL); 2397 break; 2398 2399 case CPU_DOWN_PREPARE: 2400 __cpufreq_remove_dev_prepare(dev); 2401 break; 2402 2403 case CPU_POST_DEAD: 2404 __cpufreq_remove_dev_finish(dev); 2405 break; 2406 2407 case CPU_DOWN_FAILED: 2408 cpufreq_add_dev(dev, NULL); 2409 break; 2410 } 2411 } 2412 return NOTIFY_OK; 2413 } 2414 2415 static struct notifier_block __refdata cpufreq_cpu_notifier = { 2416 .notifier_call = cpufreq_cpu_callback, 2417 }; 2418 2419 /********************************************************************* 2420 * BOOST * 2421 *********************************************************************/ 2422 static int cpufreq_boost_set_sw(int state) 2423 { 2424 struct cpufreq_frequency_table *freq_table; 2425 struct cpufreq_policy *policy; 2426 int ret = -EINVAL; 2427 2428 for_each_active_policy(policy) { 2429 freq_table = cpufreq_frequency_get_table(policy->cpu); 2430 if (freq_table) { 2431 ret = cpufreq_frequency_table_cpuinfo(policy, 2432 freq_table); 2433 if (ret) { 2434 pr_err("%s: Policy frequency update failed\n", 2435 __func__); 2436 break; 2437 } 2438 policy->user_policy.max = policy->max; 2439 __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); 2440 } 2441 } 2442 2443 return ret; 2444 } 2445 2446 int cpufreq_boost_trigger_state(int state) 2447 { 2448 unsigned long flags; 2449 int ret = 0; 2450 2451 if (cpufreq_driver->boost_enabled == state) 2452 return 0; 2453 2454 write_lock_irqsave(&cpufreq_driver_lock, flags); 2455 cpufreq_driver->boost_enabled = state; 2456 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2457 2458 ret = cpufreq_driver->set_boost(state); 2459 if (ret) { 2460 write_lock_irqsave(&cpufreq_driver_lock, flags); 2461 cpufreq_driver->boost_enabled = !state; 2462 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2463 2464 pr_err("%s: Cannot %s BOOST\n", 2465 __func__, state ? "enable" : "disable"); 2466 } 2467 2468 return ret; 2469 } 2470 2471 int cpufreq_boost_supported(void) 2472 { 2473 if (likely(cpufreq_driver)) 2474 return cpufreq_driver->boost_supported; 2475 2476 return 0; 2477 } 2478 EXPORT_SYMBOL_GPL(cpufreq_boost_supported); 2479 2480 int cpufreq_boost_enabled(void) 2481 { 2482 return cpufreq_driver->boost_enabled; 2483 } 2484 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled); 2485 2486 /********************************************************************* 2487 * REGISTER / UNREGISTER CPUFREQ DRIVER * 2488 *********************************************************************/ 2489 2490 /** 2491 * cpufreq_register_driver - register a CPU Frequency driver 2492 * @driver_data: A struct cpufreq_driver containing the values# 2493 * submitted by the CPU Frequency driver. 2494 * 2495 * Registers a CPU Frequency driver to this core code. This code 2496 * returns zero on success, -EBUSY when another driver got here first 2497 * (and isn't unregistered in the meantime). 2498 * 2499 */ 2500 int cpufreq_register_driver(struct cpufreq_driver *driver_data) 2501 { 2502 unsigned long flags; 2503 int ret; 2504 2505 if (cpufreq_disabled()) 2506 return -ENODEV; 2507 2508 if (!driver_data || !driver_data->verify || !driver_data->init || 2509 !(driver_data->setpolicy || driver_data->target_index || 2510 driver_data->target) || 2511 (driver_data->setpolicy && (driver_data->target_index || 2512 driver_data->target)) || 2513 (!!driver_data->get_intermediate != !!driver_data->target_intermediate)) 2514 return -EINVAL; 2515 2516 pr_debug("trying to register driver %s\n", driver_data->name); 2517 2518 write_lock_irqsave(&cpufreq_driver_lock, flags); 2519 if (cpufreq_driver) { 2520 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2521 return -EEXIST; 2522 } 2523 cpufreq_driver = driver_data; 2524 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2525 2526 if (driver_data->setpolicy) 2527 driver_data->flags |= CPUFREQ_CONST_LOOPS; 2528 2529 if (cpufreq_boost_supported()) { 2530 /* 2531 * Check if driver provides function to enable boost - 2532 * if not, use cpufreq_boost_set_sw as default 2533 */ 2534 if (!cpufreq_driver->set_boost) 2535 cpufreq_driver->set_boost = cpufreq_boost_set_sw; 2536 2537 ret = cpufreq_sysfs_create_file(&boost.attr); 2538 if (ret) { 2539 pr_err("%s: cannot register global BOOST sysfs file\n", 2540 __func__); 2541 goto err_null_driver; 2542 } 2543 } 2544 2545 ret = subsys_interface_register(&cpufreq_interface); 2546 if (ret) 2547 goto err_boost_unreg; 2548 2549 if (!(cpufreq_driver->flags & CPUFREQ_STICKY) && 2550 list_empty(&cpufreq_policy_list)) { 2551 /* if all ->init() calls failed, unregister */ 2552 pr_debug("%s: No CPU initialized for driver %s\n", __func__, 2553 driver_data->name); 2554 goto err_if_unreg; 2555 } 2556 2557 register_hotcpu_notifier(&cpufreq_cpu_notifier); 2558 pr_debug("driver %s up and running\n", driver_data->name); 2559 2560 return 0; 2561 err_if_unreg: 2562 subsys_interface_unregister(&cpufreq_interface); 2563 err_boost_unreg: 2564 if (cpufreq_boost_supported()) 2565 cpufreq_sysfs_remove_file(&boost.attr); 2566 err_null_driver: 2567 write_lock_irqsave(&cpufreq_driver_lock, flags); 2568 cpufreq_driver = NULL; 2569 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2570 return ret; 2571 } 2572 EXPORT_SYMBOL_GPL(cpufreq_register_driver); 2573 2574 /** 2575 * cpufreq_unregister_driver - unregister the current CPUFreq driver 2576 * 2577 * Unregister the current CPUFreq driver. Only call this if you have 2578 * the right to do so, i.e. if you have succeeded in initialising before! 2579 * Returns zero if successful, and -EINVAL if the cpufreq_driver is 2580 * currently not initialised. 2581 */ 2582 int cpufreq_unregister_driver(struct cpufreq_driver *driver) 2583 { 2584 unsigned long flags; 2585 2586 if (!cpufreq_driver || (driver != cpufreq_driver)) 2587 return -EINVAL; 2588 2589 pr_debug("unregistering driver %s\n", driver->name); 2590 2591 subsys_interface_unregister(&cpufreq_interface); 2592 if (cpufreq_boost_supported()) 2593 cpufreq_sysfs_remove_file(&boost.attr); 2594 2595 unregister_hotcpu_notifier(&cpufreq_cpu_notifier); 2596 2597 down_write(&cpufreq_rwsem); 2598 write_lock_irqsave(&cpufreq_driver_lock, flags); 2599 2600 cpufreq_driver = NULL; 2601 2602 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2603 up_write(&cpufreq_rwsem); 2604 2605 return 0; 2606 } 2607 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver); 2608 2609 /* 2610 * Stop cpufreq at shutdown to make sure it isn't holding any locks 2611 * or mutexes when secondary CPUs are halted. 2612 */ 2613 static struct syscore_ops cpufreq_syscore_ops = { 2614 .shutdown = cpufreq_suspend, 2615 }; 2616 2617 static int __init cpufreq_core_init(void) 2618 { 2619 if (cpufreq_disabled()) 2620 return -ENODEV; 2621 2622 cpufreq_global_kobject = kobject_create(); 2623 BUG_ON(!cpufreq_global_kobject); 2624 2625 register_syscore_ops(&cpufreq_syscore_ops); 2626 2627 return 0; 2628 } 2629 core_initcall(cpufreq_core_init); 2630