1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/drivers/cpufreq/cpufreq.c 4 * 5 * Copyright (C) 2001 Russell King 6 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de> 7 * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org> 8 * 9 * Oct 2005 - Ashok Raj <ashok.raj@intel.com> 10 * Added handling for CPU hotplug 11 * Feb 2006 - Jacob Shin <jacob.shin@amd.com> 12 * Fix handling for CPU hotplug -- affected CPUs 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 16 17 #include <linux/cpu.h> 18 #include <linux/cpufreq.h> 19 #include <linux/cpu_cooling.h> 20 #include <linux/delay.h> 21 #include <linux/device.h> 22 #include <linux/init.h> 23 #include <linux/kernel_stat.h> 24 #include <linux/module.h> 25 #include <linux/mutex.h> 26 #include <linux/slab.h> 27 #include <linux/suspend.h> 28 #include <linux/syscore_ops.h> 29 #include <linux/tick.h> 30 #include <trace/events/power.h> 31 32 static LIST_HEAD(cpufreq_policy_list); 33 34 /* Macros to iterate over CPU policies */ 35 #define for_each_suitable_policy(__policy, __active) \ 36 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \ 37 if ((__active) == !policy_is_inactive(__policy)) 38 39 #define for_each_active_policy(__policy) \ 40 for_each_suitable_policy(__policy, true) 41 #define for_each_inactive_policy(__policy) \ 42 for_each_suitable_policy(__policy, false) 43 44 #define for_each_policy(__policy) \ 45 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) 46 47 /* Iterate over governors */ 48 static LIST_HEAD(cpufreq_governor_list); 49 #define for_each_governor(__governor) \ 50 list_for_each_entry(__governor, &cpufreq_governor_list, governor_list) 51 52 /** 53 * The "cpufreq driver" - the arch- or hardware-dependent low 54 * level driver of CPUFreq support, and its spinlock. This lock 55 * also protects the cpufreq_cpu_data array. 56 */ 57 static struct cpufreq_driver *cpufreq_driver; 58 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data); 59 static DEFINE_RWLOCK(cpufreq_driver_lock); 60 61 /* Flag to suspend/resume CPUFreq governors */ 62 static bool cpufreq_suspended; 63 64 static inline bool has_target(void) 65 { 66 return cpufreq_driver->target_index || cpufreq_driver->target; 67 } 68 69 /* internal prototypes */ 70 static unsigned int __cpufreq_get(struct cpufreq_policy *policy); 71 static int cpufreq_init_governor(struct cpufreq_policy *policy); 72 static void cpufreq_exit_governor(struct cpufreq_policy *policy); 73 static int cpufreq_start_governor(struct cpufreq_policy *policy); 74 static void cpufreq_stop_governor(struct cpufreq_policy *policy); 75 static void cpufreq_governor_limits(struct cpufreq_policy *policy); 76 77 /** 78 * Two notifier lists: the "policy" list is involved in the 79 * validation process for a new CPU frequency policy; the 80 * "transition" list for kernel code that needs to handle 81 * changes to devices when the CPU clock speed changes. 82 * The mutex locks both lists. 83 */ 84 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list); 85 SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list); 86 87 static int off __read_mostly; 88 static int cpufreq_disabled(void) 89 { 90 return off; 91 } 92 void disable_cpufreq(void) 93 { 94 off = 1; 95 } 96 static DEFINE_MUTEX(cpufreq_governor_mutex); 97 98 bool have_governor_per_policy(void) 99 { 100 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY); 101 } 102 EXPORT_SYMBOL_GPL(have_governor_per_policy); 103 104 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) 105 { 106 if (have_governor_per_policy()) 107 return &policy->kobj; 108 else 109 return cpufreq_global_kobject; 110 } 111 EXPORT_SYMBOL_GPL(get_governor_parent_kobj); 112 113 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) 114 { 115 u64 idle_time; 116 u64 cur_wall_time; 117 u64 busy_time; 118 119 cur_wall_time = jiffies64_to_nsecs(get_jiffies_64()); 120 121 busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; 122 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; 123 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; 124 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; 125 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; 126 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; 127 128 idle_time = cur_wall_time - busy_time; 129 if (wall) 130 *wall = div_u64(cur_wall_time, NSEC_PER_USEC); 131 132 return div_u64(idle_time, NSEC_PER_USEC); 133 } 134 135 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) 136 { 137 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL); 138 139 if (idle_time == -1ULL) 140 return get_cpu_idle_time_jiffy(cpu, wall); 141 else if (!io_busy) 142 idle_time += get_cpu_iowait_time_us(cpu, wall); 143 144 return idle_time; 145 } 146 EXPORT_SYMBOL_GPL(get_cpu_idle_time); 147 148 __weak void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, 149 unsigned long max_freq) 150 { 151 } 152 EXPORT_SYMBOL_GPL(arch_set_freq_scale); 153 154 /* 155 * This is a generic cpufreq init() routine which can be used by cpufreq 156 * drivers of SMP systems. It will do following: 157 * - validate & show freq table passed 158 * - set policies transition latency 159 * - policy->cpus with all possible CPUs 160 */ 161 int cpufreq_generic_init(struct cpufreq_policy *policy, 162 struct cpufreq_frequency_table *table, 163 unsigned int transition_latency) 164 { 165 policy->freq_table = table; 166 policy->cpuinfo.transition_latency = transition_latency; 167 168 /* 169 * The driver only supports the SMP configuration where all processors 170 * share the clock and voltage and clock. 171 */ 172 cpumask_setall(policy->cpus); 173 174 return 0; 175 } 176 EXPORT_SYMBOL_GPL(cpufreq_generic_init); 177 178 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu) 179 { 180 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 181 182 return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL; 183 } 184 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw); 185 186 unsigned int cpufreq_generic_get(unsigned int cpu) 187 { 188 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); 189 190 if (!policy || IS_ERR(policy->clk)) { 191 pr_err("%s: No %s associated to cpu: %d\n", 192 __func__, policy ? "clk" : "policy", cpu); 193 return 0; 194 } 195 196 return clk_get_rate(policy->clk) / 1000; 197 } 198 EXPORT_SYMBOL_GPL(cpufreq_generic_get); 199 200 /** 201 * cpufreq_cpu_get - Return policy for a CPU and mark it as busy. 202 * @cpu: CPU to find the policy for. 203 * 204 * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment 205 * the kobject reference counter of that policy. Return a valid policy on 206 * success or NULL on failure. 207 * 208 * The policy returned by this function has to be released with the help of 209 * cpufreq_cpu_put() to balance its kobject reference counter properly. 210 */ 211 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) 212 { 213 struct cpufreq_policy *policy = NULL; 214 unsigned long flags; 215 216 if (WARN_ON(cpu >= nr_cpu_ids)) 217 return NULL; 218 219 /* get the cpufreq driver */ 220 read_lock_irqsave(&cpufreq_driver_lock, flags); 221 222 if (cpufreq_driver) { 223 /* get the CPU */ 224 policy = cpufreq_cpu_get_raw(cpu); 225 if (policy) 226 kobject_get(&policy->kobj); 227 } 228 229 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 230 231 return policy; 232 } 233 EXPORT_SYMBOL_GPL(cpufreq_cpu_get); 234 235 /** 236 * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy. 237 * @policy: cpufreq policy returned by cpufreq_cpu_get(). 238 */ 239 void cpufreq_cpu_put(struct cpufreq_policy *policy) 240 { 241 kobject_put(&policy->kobj); 242 } 243 EXPORT_SYMBOL_GPL(cpufreq_cpu_put); 244 245 /** 246 * cpufreq_cpu_release - Unlock a policy and decrement its usage counter. 247 * @policy: cpufreq policy returned by cpufreq_cpu_acquire(). 248 */ 249 void cpufreq_cpu_release(struct cpufreq_policy *policy) 250 { 251 if (WARN_ON(!policy)) 252 return; 253 254 lockdep_assert_held(&policy->rwsem); 255 256 up_write(&policy->rwsem); 257 258 cpufreq_cpu_put(policy); 259 } 260 261 /** 262 * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it. 263 * @cpu: CPU to find the policy for. 264 * 265 * Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and 266 * if the policy returned by it is not NULL, acquire its rwsem for writing. 267 * Return the policy if it is active or release it and return NULL otherwise. 268 * 269 * The policy returned by this function has to be released with the help of 270 * cpufreq_cpu_release() in order to release its rwsem and balance its usage 271 * counter properly. 272 */ 273 struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu) 274 { 275 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 276 277 if (!policy) 278 return NULL; 279 280 down_write(&policy->rwsem); 281 282 if (policy_is_inactive(policy)) { 283 cpufreq_cpu_release(policy); 284 return NULL; 285 } 286 287 return policy; 288 } 289 290 /********************************************************************* 291 * EXTERNALLY AFFECTING FREQUENCY CHANGES * 292 *********************************************************************/ 293 294 /** 295 * adjust_jiffies - adjust the system "loops_per_jiffy" 296 * 297 * This function alters the system "loops_per_jiffy" for the clock 298 * speed change. Note that loops_per_jiffy cannot be updated on SMP 299 * systems as each CPU might be scaled differently. So, use the arch 300 * per-CPU loops_per_jiffy value wherever possible. 301 */ 302 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci) 303 { 304 #ifndef CONFIG_SMP 305 static unsigned long l_p_j_ref; 306 static unsigned int l_p_j_ref_freq; 307 308 if (ci->flags & CPUFREQ_CONST_LOOPS) 309 return; 310 311 if (!l_p_j_ref_freq) { 312 l_p_j_ref = loops_per_jiffy; 313 l_p_j_ref_freq = ci->old; 314 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n", 315 l_p_j_ref, l_p_j_ref_freq); 316 } 317 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) { 318 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq, 319 ci->new); 320 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n", 321 loops_per_jiffy, ci->new); 322 } 323 #endif 324 } 325 326 /** 327 * cpufreq_notify_transition - Notify frequency transition and adjust_jiffies. 328 * @policy: cpufreq policy to enable fast frequency switching for. 329 * @freqs: contain details of the frequency update. 330 * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE. 331 * 332 * This function calls the transition notifiers and the "adjust_jiffies" 333 * function. It is called twice on all CPU frequency changes that have 334 * external effects. 335 */ 336 static void cpufreq_notify_transition(struct cpufreq_policy *policy, 337 struct cpufreq_freqs *freqs, 338 unsigned int state) 339 { 340 int cpu; 341 342 BUG_ON(irqs_disabled()); 343 344 if (cpufreq_disabled()) 345 return; 346 347 freqs->policy = policy; 348 freqs->flags = cpufreq_driver->flags; 349 pr_debug("notification %u of frequency transition to %u kHz\n", 350 state, freqs->new); 351 352 switch (state) { 353 case CPUFREQ_PRECHANGE: 354 /* 355 * Detect if the driver reported a value as "old frequency" 356 * which is not equal to what the cpufreq core thinks is 357 * "old frequency". 358 */ 359 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) { 360 if (policy->cur && (policy->cur != freqs->old)) { 361 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n", 362 freqs->old, policy->cur); 363 freqs->old = policy->cur; 364 } 365 } 366 367 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 368 CPUFREQ_PRECHANGE, freqs); 369 370 adjust_jiffies(CPUFREQ_PRECHANGE, freqs); 371 break; 372 373 case CPUFREQ_POSTCHANGE: 374 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs); 375 pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new, 376 cpumask_pr_args(policy->cpus)); 377 378 for_each_cpu(cpu, policy->cpus) 379 trace_cpu_frequency(freqs->new, cpu); 380 381 srcu_notifier_call_chain(&cpufreq_transition_notifier_list, 382 CPUFREQ_POSTCHANGE, freqs); 383 384 cpufreq_stats_record_transition(policy, freqs->new); 385 policy->cur = freqs->new; 386 } 387 } 388 389 /* Do post notifications when there are chances that transition has failed */ 390 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy, 391 struct cpufreq_freqs *freqs, int transition_failed) 392 { 393 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 394 if (!transition_failed) 395 return; 396 397 swap(freqs->old, freqs->new); 398 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 399 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE); 400 } 401 402 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy, 403 struct cpufreq_freqs *freqs) 404 { 405 406 /* 407 * Catch double invocations of _begin() which lead to self-deadlock. 408 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core 409 * doesn't invoke _begin() on their behalf, and hence the chances of 410 * double invocations are very low. Moreover, there are scenarios 411 * where these checks can emit false-positive warnings in these 412 * drivers; so we avoid that by skipping them altogether. 413 */ 414 WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION) 415 && current == policy->transition_task); 416 417 wait: 418 wait_event(policy->transition_wait, !policy->transition_ongoing); 419 420 spin_lock(&policy->transition_lock); 421 422 if (unlikely(policy->transition_ongoing)) { 423 spin_unlock(&policy->transition_lock); 424 goto wait; 425 } 426 427 policy->transition_ongoing = true; 428 policy->transition_task = current; 429 430 spin_unlock(&policy->transition_lock); 431 432 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE); 433 } 434 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin); 435 436 void cpufreq_freq_transition_end(struct cpufreq_policy *policy, 437 struct cpufreq_freqs *freqs, int transition_failed) 438 { 439 if (WARN_ON(!policy->transition_ongoing)) 440 return; 441 442 cpufreq_notify_post_transition(policy, freqs, transition_failed); 443 444 policy->transition_ongoing = false; 445 policy->transition_task = NULL; 446 447 wake_up(&policy->transition_wait); 448 } 449 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end); 450 451 /* 452 * Fast frequency switching status count. Positive means "enabled", negative 453 * means "disabled" and 0 means "not decided yet". 454 */ 455 static int cpufreq_fast_switch_count; 456 static DEFINE_MUTEX(cpufreq_fast_switch_lock); 457 458 static void cpufreq_list_transition_notifiers(void) 459 { 460 struct notifier_block *nb; 461 462 pr_info("Registered transition notifiers:\n"); 463 464 mutex_lock(&cpufreq_transition_notifier_list.mutex); 465 466 for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next) 467 pr_info("%pS\n", nb->notifier_call); 468 469 mutex_unlock(&cpufreq_transition_notifier_list.mutex); 470 } 471 472 /** 473 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy. 474 * @policy: cpufreq policy to enable fast frequency switching for. 475 * 476 * Try to enable fast frequency switching for @policy. 477 * 478 * The attempt will fail if there is at least one transition notifier registered 479 * at this point, as fast frequency switching is quite fundamentally at odds 480 * with transition notifiers. Thus if successful, it will make registration of 481 * transition notifiers fail going forward. 482 */ 483 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy) 484 { 485 lockdep_assert_held(&policy->rwsem); 486 487 if (!policy->fast_switch_possible) 488 return; 489 490 mutex_lock(&cpufreq_fast_switch_lock); 491 if (cpufreq_fast_switch_count >= 0) { 492 cpufreq_fast_switch_count++; 493 policy->fast_switch_enabled = true; 494 } else { 495 pr_warn("CPU%u: Fast frequency switching not enabled\n", 496 policy->cpu); 497 cpufreq_list_transition_notifiers(); 498 } 499 mutex_unlock(&cpufreq_fast_switch_lock); 500 } 501 EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch); 502 503 /** 504 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy. 505 * @policy: cpufreq policy to disable fast frequency switching for. 506 */ 507 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy) 508 { 509 mutex_lock(&cpufreq_fast_switch_lock); 510 if (policy->fast_switch_enabled) { 511 policy->fast_switch_enabled = false; 512 if (!WARN_ON(cpufreq_fast_switch_count <= 0)) 513 cpufreq_fast_switch_count--; 514 } 515 mutex_unlock(&cpufreq_fast_switch_lock); 516 } 517 EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch); 518 519 /** 520 * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported 521 * one. 522 * @target_freq: target frequency to resolve. 523 * 524 * The target to driver frequency mapping is cached in the policy. 525 * 526 * Return: Lowest driver-supported frequency greater than or equal to the 527 * given target_freq, subject to policy (min/max) and driver limitations. 528 */ 529 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy, 530 unsigned int target_freq) 531 { 532 target_freq = clamp_val(target_freq, policy->min, policy->max); 533 policy->cached_target_freq = target_freq; 534 535 if (cpufreq_driver->target_index) { 536 int idx; 537 538 idx = cpufreq_frequency_table_target(policy, target_freq, 539 CPUFREQ_RELATION_L); 540 policy->cached_resolved_idx = idx; 541 return policy->freq_table[idx].frequency; 542 } 543 544 if (cpufreq_driver->resolve_freq) 545 return cpufreq_driver->resolve_freq(policy, target_freq); 546 547 return target_freq; 548 } 549 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq); 550 551 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy) 552 { 553 unsigned int latency; 554 555 if (policy->transition_delay_us) 556 return policy->transition_delay_us; 557 558 latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC; 559 if (latency) { 560 /* 561 * For platforms that can change the frequency very fast (< 10 562 * us), the above formula gives a decent transition delay. But 563 * for platforms where transition_latency is in milliseconds, it 564 * ends up giving unrealistic values. 565 * 566 * Cap the default transition delay to 10 ms, which seems to be 567 * a reasonable amount of time after which we should reevaluate 568 * the frequency. 569 */ 570 return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000); 571 } 572 573 return LATENCY_MULTIPLIER; 574 } 575 EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us); 576 577 /********************************************************************* 578 * SYSFS INTERFACE * 579 *********************************************************************/ 580 static ssize_t show_boost(struct kobject *kobj, 581 struct kobj_attribute *attr, char *buf) 582 { 583 return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled); 584 } 585 586 static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr, 587 const char *buf, size_t count) 588 { 589 int ret, enable; 590 591 ret = sscanf(buf, "%d", &enable); 592 if (ret != 1 || enable < 0 || enable > 1) 593 return -EINVAL; 594 595 if (cpufreq_boost_trigger_state(enable)) { 596 pr_err("%s: Cannot %s BOOST!\n", 597 __func__, enable ? "enable" : "disable"); 598 return -EINVAL; 599 } 600 601 pr_debug("%s: cpufreq BOOST %s\n", 602 __func__, enable ? "enabled" : "disabled"); 603 604 return count; 605 } 606 define_one_global_rw(boost); 607 608 static struct cpufreq_governor *find_governor(const char *str_governor) 609 { 610 struct cpufreq_governor *t; 611 612 for_each_governor(t) 613 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN)) 614 return t; 615 616 return NULL; 617 } 618 619 static int cpufreq_parse_policy(char *str_governor, 620 struct cpufreq_policy *policy) 621 { 622 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) { 623 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 624 return 0; 625 } 626 if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN)) { 627 policy->policy = CPUFREQ_POLICY_POWERSAVE; 628 return 0; 629 } 630 return -EINVAL; 631 } 632 633 /** 634 * cpufreq_parse_governor - parse a governor string only for !setpolicy 635 */ 636 static int cpufreq_parse_governor(char *str_governor, 637 struct cpufreq_policy *policy) 638 { 639 struct cpufreq_governor *t; 640 641 mutex_lock(&cpufreq_governor_mutex); 642 643 t = find_governor(str_governor); 644 if (!t) { 645 int ret; 646 647 mutex_unlock(&cpufreq_governor_mutex); 648 649 ret = request_module("cpufreq_%s", str_governor); 650 if (ret) 651 return -EINVAL; 652 653 mutex_lock(&cpufreq_governor_mutex); 654 655 t = find_governor(str_governor); 656 } 657 if (t && !try_module_get(t->owner)) 658 t = NULL; 659 660 mutex_unlock(&cpufreq_governor_mutex); 661 662 if (t) { 663 policy->governor = t; 664 return 0; 665 } 666 667 return -EINVAL; 668 } 669 670 /** 671 * cpufreq_per_cpu_attr_read() / show_##file_name() - 672 * print out cpufreq information 673 * 674 * Write out information from cpufreq_driver->policy[cpu]; object must be 675 * "unsigned int". 676 */ 677 678 #define show_one(file_name, object) \ 679 static ssize_t show_##file_name \ 680 (struct cpufreq_policy *policy, char *buf) \ 681 { \ 682 return sprintf(buf, "%u\n", policy->object); \ 683 } 684 685 show_one(cpuinfo_min_freq, cpuinfo.min_freq); 686 show_one(cpuinfo_max_freq, cpuinfo.max_freq); 687 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency); 688 show_one(scaling_min_freq, min); 689 show_one(scaling_max_freq, max); 690 691 __weak unsigned int arch_freq_get_on_cpu(int cpu) 692 { 693 return 0; 694 } 695 696 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf) 697 { 698 ssize_t ret; 699 unsigned int freq; 700 701 freq = arch_freq_get_on_cpu(policy->cpu); 702 if (freq) 703 ret = sprintf(buf, "%u\n", freq); 704 else if (cpufreq_driver && cpufreq_driver->setpolicy && 705 cpufreq_driver->get) 706 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu)); 707 else 708 ret = sprintf(buf, "%u\n", policy->cur); 709 return ret; 710 } 711 712 /** 713 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access 714 */ 715 #define store_one(file_name, object) \ 716 static ssize_t store_##file_name \ 717 (struct cpufreq_policy *policy, const char *buf, size_t count) \ 718 { \ 719 int ret, temp; \ 720 struct cpufreq_policy new_policy; \ 721 \ 722 memcpy(&new_policy, policy, sizeof(*policy)); \ 723 new_policy.min = policy->user_policy.min; \ 724 new_policy.max = policy->user_policy.max; \ 725 \ 726 ret = sscanf(buf, "%u", &new_policy.object); \ 727 if (ret != 1) \ 728 return -EINVAL; \ 729 \ 730 temp = new_policy.object; \ 731 ret = cpufreq_set_policy(policy, &new_policy); \ 732 if (!ret) \ 733 policy->user_policy.object = temp; \ 734 \ 735 return ret ? ret : count; \ 736 } 737 738 store_one(scaling_min_freq, min); 739 store_one(scaling_max_freq, max); 740 741 /** 742 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware 743 */ 744 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy, 745 char *buf) 746 { 747 unsigned int cur_freq = __cpufreq_get(policy); 748 749 if (cur_freq) 750 return sprintf(buf, "%u\n", cur_freq); 751 752 return sprintf(buf, "<unknown>\n"); 753 } 754 755 /** 756 * show_scaling_governor - show the current policy for the specified CPU 757 */ 758 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf) 759 { 760 if (policy->policy == CPUFREQ_POLICY_POWERSAVE) 761 return sprintf(buf, "powersave\n"); 762 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) 763 return sprintf(buf, "performance\n"); 764 else if (policy->governor) 765 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", 766 policy->governor->name); 767 return -EINVAL; 768 } 769 770 /** 771 * store_scaling_governor - store policy for the specified CPU 772 */ 773 static ssize_t store_scaling_governor(struct cpufreq_policy *policy, 774 const char *buf, size_t count) 775 { 776 int ret; 777 char str_governor[16]; 778 struct cpufreq_policy new_policy; 779 780 memcpy(&new_policy, policy, sizeof(*policy)); 781 782 ret = sscanf(buf, "%15s", str_governor); 783 if (ret != 1) 784 return -EINVAL; 785 786 if (cpufreq_driver->setpolicy) { 787 if (cpufreq_parse_policy(str_governor, &new_policy)) 788 return -EINVAL; 789 } else { 790 if (cpufreq_parse_governor(str_governor, &new_policy)) 791 return -EINVAL; 792 } 793 794 ret = cpufreq_set_policy(policy, &new_policy); 795 796 if (new_policy.governor) 797 module_put(new_policy.governor->owner); 798 799 return ret ? ret : count; 800 } 801 802 /** 803 * show_scaling_driver - show the cpufreq driver currently loaded 804 */ 805 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf) 806 { 807 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name); 808 } 809 810 /** 811 * show_scaling_available_governors - show the available CPUfreq governors 812 */ 813 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy, 814 char *buf) 815 { 816 ssize_t i = 0; 817 struct cpufreq_governor *t; 818 819 if (!has_target()) { 820 i += sprintf(buf, "performance powersave"); 821 goto out; 822 } 823 824 for_each_governor(t) { 825 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) 826 - (CPUFREQ_NAME_LEN + 2))) 827 goto out; 828 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name); 829 } 830 out: 831 i += sprintf(&buf[i], "\n"); 832 return i; 833 } 834 835 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf) 836 { 837 ssize_t i = 0; 838 unsigned int cpu; 839 840 for_each_cpu(cpu, mask) { 841 if (i) 842 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " "); 843 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu); 844 if (i >= (PAGE_SIZE - 5)) 845 break; 846 } 847 i += sprintf(&buf[i], "\n"); 848 return i; 849 } 850 EXPORT_SYMBOL_GPL(cpufreq_show_cpus); 851 852 /** 853 * show_related_cpus - show the CPUs affected by each transition even if 854 * hw coordination is in use 855 */ 856 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf) 857 { 858 return cpufreq_show_cpus(policy->related_cpus, buf); 859 } 860 861 /** 862 * show_affected_cpus - show the CPUs affected by each transition 863 */ 864 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf) 865 { 866 return cpufreq_show_cpus(policy->cpus, buf); 867 } 868 869 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy, 870 const char *buf, size_t count) 871 { 872 unsigned int freq = 0; 873 unsigned int ret; 874 875 if (!policy->governor || !policy->governor->store_setspeed) 876 return -EINVAL; 877 878 ret = sscanf(buf, "%u", &freq); 879 if (ret != 1) 880 return -EINVAL; 881 882 policy->governor->store_setspeed(policy, freq); 883 884 return count; 885 } 886 887 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf) 888 { 889 if (!policy->governor || !policy->governor->show_setspeed) 890 return sprintf(buf, "<unsupported>\n"); 891 892 return policy->governor->show_setspeed(policy, buf); 893 } 894 895 /** 896 * show_bios_limit - show the current cpufreq HW/BIOS limitation 897 */ 898 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf) 899 { 900 unsigned int limit; 901 int ret; 902 ret = cpufreq_driver->bios_limit(policy->cpu, &limit); 903 if (!ret) 904 return sprintf(buf, "%u\n", limit); 905 return sprintf(buf, "%u\n", policy->cpuinfo.max_freq); 906 } 907 908 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400); 909 cpufreq_freq_attr_ro(cpuinfo_min_freq); 910 cpufreq_freq_attr_ro(cpuinfo_max_freq); 911 cpufreq_freq_attr_ro(cpuinfo_transition_latency); 912 cpufreq_freq_attr_ro(scaling_available_governors); 913 cpufreq_freq_attr_ro(scaling_driver); 914 cpufreq_freq_attr_ro(scaling_cur_freq); 915 cpufreq_freq_attr_ro(bios_limit); 916 cpufreq_freq_attr_ro(related_cpus); 917 cpufreq_freq_attr_ro(affected_cpus); 918 cpufreq_freq_attr_rw(scaling_min_freq); 919 cpufreq_freq_attr_rw(scaling_max_freq); 920 cpufreq_freq_attr_rw(scaling_governor); 921 cpufreq_freq_attr_rw(scaling_setspeed); 922 923 static struct attribute *default_attrs[] = { 924 &cpuinfo_min_freq.attr, 925 &cpuinfo_max_freq.attr, 926 &cpuinfo_transition_latency.attr, 927 &scaling_min_freq.attr, 928 &scaling_max_freq.attr, 929 &affected_cpus.attr, 930 &related_cpus.attr, 931 &scaling_governor.attr, 932 &scaling_driver.attr, 933 &scaling_available_governors.attr, 934 &scaling_setspeed.attr, 935 NULL 936 }; 937 938 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj) 939 #define to_attr(a) container_of(a, struct freq_attr, attr) 940 941 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) 942 { 943 struct cpufreq_policy *policy = to_policy(kobj); 944 struct freq_attr *fattr = to_attr(attr); 945 ssize_t ret; 946 947 down_read(&policy->rwsem); 948 ret = fattr->show(policy, buf); 949 up_read(&policy->rwsem); 950 951 return ret; 952 } 953 954 static ssize_t store(struct kobject *kobj, struct attribute *attr, 955 const char *buf, size_t count) 956 { 957 struct cpufreq_policy *policy = to_policy(kobj); 958 struct freq_attr *fattr = to_attr(attr); 959 ssize_t ret = -EINVAL; 960 961 /* 962 * cpus_read_trylock() is used here to work around a circular lock 963 * dependency problem with respect to the cpufreq_register_driver(). 964 */ 965 if (!cpus_read_trylock()) 966 return -EBUSY; 967 968 if (cpu_online(policy->cpu)) { 969 down_write(&policy->rwsem); 970 ret = fattr->store(policy, buf, count); 971 up_write(&policy->rwsem); 972 } 973 974 cpus_read_unlock(); 975 976 return ret; 977 } 978 979 static void cpufreq_sysfs_release(struct kobject *kobj) 980 { 981 struct cpufreq_policy *policy = to_policy(kobj); 982 pr_debug("last reference is dropped\n"); 983 complete(&policy->kobj_unregister); 984 } 985 986 static const struct sysfs_ops sysfs_ops = { 987 .show = show, 988 .store = store, 989 }; 990 991 static struct kobj_type ktype_cpufreq = { 992 .sysfs_ops = &sysfs_ops, 993 .default_attrs = default_attrs, 994 .release = cpufreq_sysfs_release, 995 }; 996 997 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu) 998 { 999 struct device *dev = get_cpu_device(cpu); 1000 1001 if (!dev) 1002 return; 1003 1004 if (cpumask_test_and_set_cpu(cpu, policy->real_cpus)) 1005 return; 1006 1007 dev_dbg(dev, "%s: Adding symlink\n", __func__); 1008 if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq")) 1009 dev_err(dev, "cpufreq symlink creation failed\n"); 1010 } 1011 1012 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, 1013 struct device *dev) 1014 { 1015 dev_dbg(dev, "%s: Removing symlink\n", __func__); 1016 sysfs_remove_link(&dev->kobj, "cpufreq"); 1017 } 1018 1019 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy) 1020 { 1021 struct freq_attr **drv_attr; 1022 int ret = 0; 1023 1024 /* set up files for this cpu device */ 1025 drv_attr = cpufreq_driver->attr; 1026 while (drv_attr && *drv_attr) { 1027 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr)); 1028 if (ret) 1029 return ret; 1030 drv_attr++; 1031 } 1032 if (cpufreq_driver->get) { 1033 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr); 1034 if (ret) 1035 return ret; 1036 } 1037 1038 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr); 1039 if (ret) 1040 return ret; 1041 1042 if (cpufreq_driver->bios_limit) { 1043 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr); 1044 if (ret) 1045 return ret; 1046 } 1047 1048 return 0; 1049 } 1050 1051 __weak struct cpufreq_governor *cpufreq_default_governor(void) 1052 { 1053 return NULL; 1054 } 1055 1056 static int cpufreq_init_policy(struct cpufreq_policy *policy) 1057 { 1058 struct cpufreq_governor *gov = NULL, *def_gov = NULL; 1059 struct cpufreq_policy new_policy; 1060 1061 memcpy(&new_policy, policy, sizeof(*policy)); 1062 1063 def_gov = cpufreq_default_governor(); 1064 1065 if (has_target()) { 1066 /* 1067 * Update governor of new_policy to the governor used before 1068 * hotplug 1069 */ 1070 gov = find_governor(policy->last_governor); 1071 if (gov) { 1072 pr_debug("Restoring governor %s for cpu %d\n", 1073 policy->governor->name, policy->cpu); 1074 } else { 1075 if (!def_gov) 1076 return -ENODATA; 1077 gov = def_gov; 1078 } 1079 new_policy.governor = gov; 1080 } else { 1081 /* Use the default policy if there is no last_policy. */ 1082 if (policy->last_policy) { 1083 new_policy.policy = policy->last_policy; 1084 } else { 1085 if (!def_gov) 1086 return -ENODATA; 1087 cpufreq_parse_policy(def_gov->name, &new_policy); 1088 } 1089 } 1090 1091 return cpufreq_set_policy(policy, &new_policy); 1092 } 1093 1094 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu) 1095 { 1096 int ret = 0; 1097 1098 /* Has this CPU been taken care of already? */ 1099 if (cpumask_test_cpu(cpu, policy->cpus)) 1100 return 0; 1101 1102 down_write(&policy->rwsem); 1103 if (has_target()) 1104 cpufreq_stop_governor(policy); 1105 1106 cpumask_set_cpu(cpu, policy->cpus); 1107 1108 if (has_target()) { 1109 ret = cpufreq_start_governor(policy); 1110 if (ret) 1111 pr_err("%s: Failed to start governor\n", __func__); 1112 } 1113 up_write(&policy->rwsem); 1114 return ret; 1115 } 1116 1117 static void handle_update(struct work_struct *work) 1118 { 1119 struct cpufreq_policy *policy = 1120 container_of(work, struct cpufreq_policy, update); 1121 unsigned int cpu = policy->cpu; 1122 pr_debug("handle_update for cpu %u called\n", cpu); 1123 cpufreq_update_policy(cpu); 1124 } 1125 1126 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) 1127 { 1128 struct cpufreq_policy *policy; 1129 int ret; 1130 1131 policy = kzalloc(sizeof(*policy), GFP_KERNEL); 1132 if (!policy) 1133 return NULL; 1134 1135 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL)) 1136 goto err_free_policy; 1137 1138 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) 1139 goto err_free_cpumask; 1140 1141 if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL)) 1142 goto err_free_rcpumask; 1143 1144 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, 1145 cpufreq_global_kobject, "policy%u", cpu); 1146 if (ret) { 1147 pr_err("%s: failed to init policy->kobj: %d\n", __func__, ret); 1148 /* 1149 * The entire policy object will be freed below, but the extra 1150 * memory allocated for the kobject name needs to be freed by 1151 * releasing the kobject. 1152 */ 1153 kobject_put(&policy->kobj); 1154 goto err_free_real_cpus; 1155 } 1156 1157 INIT_LIST_HEAD(&policy->policy_list); 1158 init_rwsem(&policy->rwsem); 1159 spin_lock_init(&policy->transition_lock); 1160 init_waitqueue_head(&policy->transition_wait); 1161 init_completion(&policy->kobj_unregister); 1162 INIT_WORK(&policy->update, handle_update); 1163 1164 policy->cpu = cpu; 1165 return policy; 1166 1167 err_free_real_cpus: 1168 free_cpumask_var(policy->real_cpus); 1169 err_free_rcpumask: 1170 free_cpumask_var(policy->related_cpus); 1171 err_free_cpumask: 1172 free_cpumask_var(policy->cpus); 1173 err_free_policy: 1174 kfree(policy); 1175 1176 return NULL; 1177 } 1178 1179 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy) 1180 { 1181 struct kobject *kobj; 1182 struct completion *cmp; 1183 1184 down_write(&policy->rwsem); 1185 cpufreq_stats_free_table(policy); 1186 kobj = &policy->kobj; 1187 cmp = &policy->kobj_unregister; 1188 up_write(&policy->rwsem); 1189 kobject_put(kobj); 1190 1191 /* 1192 * We need to make sure that the underlying kobj is 1193 * actually not referenced anymore by anybody before we 1194 * proceed with unloading. 1195 */ 1196 pr_debug("waiting for dropping of refcount\n"); 1197 wait_for_completion(cmp); 1198 pr_debug("wait complete\n"); 1199 } 1200 1201 static void cpufreq_policy_free(struct cpufreq_policy *policy) 1202 { 1203 unsigned long flags; 1204 int cpu; 1205 1206 /* Remove policy from list */ 1207 write_lock_irqsave(&cpufreq_driver_lock, flags); 1208 list_del(&policy->policy_list); 1209 1210 for_each_cpu(cpu, policy->related_cpus) 1211 per_cpu(cpufreq_cpu_data, cpu) = NULL; 1212 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1213 1214 cpufreq_policy_put_kobj(policy); 1215 free_cpumask_var(policy->real_cpus); 1216 free_cpumask_var(policy->related_cpus); 1217 free_cpumask_var(policy->cpus); 1218 kfree(policy); 1219 } 1220 1221 static int cpufreq_online(unsigned int cpu) 1222 { 1223 struct cpufreq_policy *policy; 1224 bool new_policy; 1225 unsigned long flags; 1226 unsigned int j; 1227 int ret; 1228 1229 pr_debug("%s: bringing CPU%u online\n", __func__, cpu); 1230 1231 /* Check if this CPU already has a policy to manage it */ 1232 policy = per_cpu(cpufreq_cpu_data, cpu); 1233 if (policy) { 1234 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); 1235 if (!policy_is_inactive(policy)) 1236 return cpufreq_add_policy_cpu(policy, cpu); 1237 1238 /* This is the only online CPU for the policy. Start over. */ 1239 new_policy = false; 1240 down_write(&policy->rwsem); 1241 policy->cpu = cpu; 1242 policy->governor = NULL; 1243 up_write(&policy->rwsem); 1244 } else { 1245 new_policy = true; 1246 policy = cpufreq_policy_alloc(cpu); 1247 if (!policy) 1248 return -ENOMEM; 1249 } 1250 1251 if (!new_policy && cpufreq_driver->online) { 1252 ret = cpufreq_driver->online(policy); 1253 if (ret) { 1254 pr_debug("%s: %d: initialization failed\n", __func__, 1255 __LINE__); 1256 goto out_exit_policy; 1257 } 1258 1259 /* Recover policy->cpus using related_cpus */ 1260 cpumask_copy(policy->cpus, policy->related_cpus); 1261 } else { 1262 cpumask_copy(policy->cpus, cpumask_of(cpu)); 1263 1264 /* 1265 * Call driver. From then on the cpufreq must be able 1266 * to accept all calls to ->verify and ->setpolicy for this CPU. 1267 */ 1268 ret = cpufreq_driver->init(policy); 1269 if (ret) { 1270 pr_debug("%s: %d: initialization failed\n", __func__, 1271 __LINE__); 1272 goto out_free_policy; 1273 } 1274 1275 ret = cpufreq_table_validate_and_sort(policy); 1276 if (ret) 1277 goto out_exit_policy; 1278 1279 /* related_cpus should at least include policy->cpus. */ 1280 cpumask_copy(policy->related_cpus, policy->cpus); 1281 } 1282 1283 down_write(&policy->rwsem); 1284 /* 1285 * affected cpus must always be the one, which are online. We aren't 1286 * managing offline cpus here. 1287 */ 1288 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); 1289 1290 if (new_policy) { 1291 policy->user_policy.min = policy->min; 1292 policy->user_policy.max = policy->max; 1293 1294 for_each_cpu(j, policy->related_cpus) { 1295 per_cpu(cpufreq_cpu_data, j) = policy; 1296 add_cpu_dev_symlink(policy, j); 1297 } 1298 } else { 1299 policy->min = policy->user_policy.min; 1300 policy->max = policy->user_policy.max; 1301 } 1302 1303 if (cpufreq_driver->get && !cpufreq_driver->setpolicy) { 1304 policy->cur = cpufreq_driver->get(policy->cpu); 1305 if (!policy->cur) { 1306 pr_err("%s: ->get() failed\n", __func__); 1307 goto out_destroy_policy; 1308 } 1309 } 1310 1311 /* 1312 * Sometimes boot loaders set CPU frequency to a value outside of 1313 * frequency table present with cpufreq core. In such cases CPU might be 1314 * unstable if it has to run on that frequency for long duration of time 1315 * and so its better to set it to a frequency which is specified in 1316 * freq-table. This also makes cpufreq stats inconsistent as 1317 * cpufreq-stats would fail to register because current frequency of CPU 1318 * isn't found in freq-table. 1319 * 1320 * Because we don't want this change to effect boot process badly, we go 1321 * for the next freq which is >= policy->cur ('cur' must be set by now, 1322 * otherwise we will end up setting freq to lowest of the table as 'cur' 1323 * is initialized to zero). 1324 * 1325 * We are passing target-freq as "policy->cur - 1" otherwise 1326 * __cpufreq_driver_target() would simply fail, as policy->cur will be 1327 * equal to target-freq. 1328 */ 1329 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK) 1330 && has_target()) { 1331 /* Are we running at unknown frequency ? */ 1332 ret = cpufreq_frequency_table_get_index(policy, policy->cur); 1333 if (ret == -EINVAL) { 1334 /* Warn user and fix it */ 1335 pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n", 1336 __func__, policy->cpu, policy->cur); 1337 ret = __cpufreq_driver_target(policy, policy->cur - 1, 1338 CPUFREQ_RELATION_L); 1339 1340 /* 1341 * Reaching here after boot in a few seconds may not 1342 * mean that system will remain stable at "unknown" 1343 * frequency for longer duration. Hence, a BUG_ON(). 1344 */ 1345 BUG_ON(ret); 1346 pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n", 1347 __func__, policy->cpu, policy->cur); 1348 } 1349 } 1350 1351 if (new_policy) { 1352 ret = cpufreq_add_dev_interface(policy); 1353 if (ret) 1354 goto out_destroy_policy; 1355 1356 cpufreq_stats_create_table(policy); 1357 1358 write_lock_irqsave(&cpufreq_driver_lock, flags); 1359 list_add(&policy->policy_list, &cpufreq_policy_list); 1360 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 1361 } 1362 1363 ret = cpufreq_init_policy(policy); 1364 if (ret) { 1365 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n", 1366 __func__, cpu, ret); 1367 goto out_destroy_policy; 1368 } 1369 1370 up_write(&policy->rwsem); 1371 1372 kobject_uevent(&policy->kobj, KOBJ_ADD); 1373 1374 /* Callback for handling stuff after policy is ready */ 1375 if (cpufreq_driver->ready) 1376 cpufreq_driver->ready(policy); 1377 1378 if (IS_ENABLED(CONFIG_CPU_THERMAL) && 1379 cpufreq_driver->flags & CPUFREQ_IS_COOLING_DEV) 1380 policy->cdev = of_cpufreq_cooling_register(policy); 1381 1382 pr_debug("initialization complete\n"); 1383 1384 return 0; 1385 1386 out_destroy_policy: 1387 for_each_cpu(j, policy->real_cpus) 1388 remove_cpu_dev_symlink(policy, get_cpu_device(j)); 1389 1390 up_write(&policy->rwsem); 1391 1392 out_exit_policy: 1393 if (cpufreq_driver->exit) 1394 cpufreq_driver->exit(policy); 1395 1396 out_free_policy: 1397 cpufreq_policy_free(policy); 1398 return ret; 1399 } 1400 1401 /** 1402 * cpufreq_add_dev - the cpufreq interface for a CPU device. 1403 * @dev: CPU device. 1404 * @sif: Subsystem interface structure pointer (not used) 1405 */ 1406 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) 1407 { 1408 struct cpufreq_policy *policy; 1409 unsigned cpu = dev->id; 1410 int ret; 1411 1412 dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu); 1413 1414 if (cpu_online(cpu)) { 1415 ret = cpufreq_online(cpu); 1416 if (ret) 1417 return ret; 1418 } 1419 1420 /* Create sysfs link on CPU registration */ 1421 policy = per_cpu(cpufreq_cpu_data, cpu); 1422 if (policy) 1423 add_cpu_dev_symlink(policy, cpu); 1424 1425 return 0; 1426 } 1427 1428 static int cpufreq_offline(unsigned int cpu) 1429 { 1430 struct cpufreq_policy *policy; 1431 int ret; 1432 1433 pr_debug("%s: unregistering CPU %u\n", __func__, cpu); 1434 1435 policy = cpufreq_cpu_get_raw(cpu); 1436 if (!policy) { 1437 pr_debug("%s: No cpu_data found\n", __func__); 1438 return 0; 1439 } 1440 1441 down_write(&policy->rwsem); 1442 if (has_target()) 1443 cpufreq_stop_governor(policy); 1444 1445 cpumask_clear_cpu(cpu, policy->cpus); 1446 1447 if (policy_is_inactive(policy)) { 1448 if (has_target()) 1449 strncpy(policy->last_governor, policy->governor->name, 1450 CPUFREQ_NAME_LEN); 1451 else 1452 policy->last_policy = policy->policy; 1453 } else if (cpu == policy->cpu) { 1454 /* Nominate new CPU */ 1455 policy->cpu = cpumask_any(policy->cpus); 1456 } 1457 1458 /* Start governor again for active policy */ 1459 if (!policy_is_inactive(policy)) { 1460 if (has_target()) { 1461 ret = cpufreq_start_governor(policy); 1462 if (ret) 1463 pr_err("%s: Failed to start governor\n", __func__); 1464 } 1465 1466 goto unlock; 1467 } 1468 1469 if (IS_ENABLED(CONFIG_CPU_THERMAL) && 1470 cpufreq_driver->flags & CPUFREQ_IS_COOLING_DEV) { 1471 cpufreq_cooling_unregister(policy->cdev); 1472 policy->cdev = NULL; 1473 } 1474 1475 if (cpufreq_driver->stop_cpu) 1476 cpufreq_driver->stop_cpu(policy); 1477 1478 if (has_target()) 1479 cpufreq_exit_governor(policy); 1480 1481 /* 1482 * Perform the ->offline() during light-weight tear-down, as 1483 * that allows fast recovery when the CPU comes back. 1484 */ 1485 if (cpufreq_driver->offline) { 1486 cpufreq_driver->offline(policy); 1487 } else if (cpufreq_driver->exit) { 1488 cpufreq_driver->exit(policy); 1489 policy->freq_table = NULL; 1490 } 1491 1492 unlock: 1493 up_write(&policy->rwsem); 1494 return 0; 1495 } 1496 1497 /** 1498 * cpufreq_remove_dev - remove a CPU device 1499 * 1500 * Removes the cpufreq interface for a CPU device. 1501 */ 1502 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) 1503 { 1504 unsigned int cpu = dev->id; 1505 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); 1506 1507 if (!policy) 1508 return; 1509 1510 if (cpu_online(cpu)) 1511 cpufreq_offline(cpu); 1512 1513 cpumask_clear_cpu(cpu, policy->real_cpus); 1514 remove_cpu_dev_symlink(policy, dev); 1515 1516 if (cpumask_empty(policy->real_cpus)) { 1517 /* We did light-weight exit earlier, do full tear down now */ 1518 if (cpufreq_driver->offline) 1519 cpufreq_driver->exit(policy); 1520 1521 cpufreq_policy_free(policy); 1522 } 1523 } 1524 1525 /** 1526 * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're 1527 * in deep trouble. 1528 * @policy: policy managing CPUs 1529 * @new_freq: CPU frequency the CPU actually runs at 1530 * 1531 * We adjust to current frequency first, and need to clean up later. 1532 * So either call to cpufreq_update_policy() or schedule handle_update()). 1533 */ 1534 static void cpufreq_out_of_sync(struct cpufreq_policy *policy, 1535 unsigned int new_freq) 1536 { 1537 struct cpufreq_freqs freqs; 1538 1539 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n", 1540 policy->cur, new_freq); 1541 1542 freqs.old = policy->cur; 1543 freqs.new = new_freq; 1544 1545 cpufreq_freq_transition_begin(policy, &freqs); 1546 cpufreq_freq_transition_end(policy, &freqs, 0); 1547 } 1548 1549 /** 1550 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur 1551 * @cpu: CPU number 1552 * 1553 * This is the last known freq, without actually getting it from the driver. 1554 * Return value will be same as what is shown in scaling_cur_freq in sysfs. 1555 */ 1556 unsigned int cpufreq_quick_get(unsigned int cpu) 1557 { 1558 struct cpufreq_policy *policy; 1559 unsigned int ret_freq = 0; 1560 unsigned long flags; 1561 1562 read_lock_irqsave(&cpufreq_driver_lock, flags); 1563 1564 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) { 1565 ret_freq = cpufreq_driver->get(cpu); 1566 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1567 return ret_freq; 1568 } 1569 1570 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 1571 1572 policy = cpufreq_cpu_get(cpu); 1573 if (policy) { 1574 ret_freq = policy->cur; 1575 cpufreq_cpu_put(policy); 1576 } 1577 1578 return ret_freq; 1579 } 1580 EXPORT_SYMBOL(cpufreq_quick_get); 1581 1582 /** 1583 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU 1584 * @cpu: CPU number 1585 * 1586 * Just return the max possible frequency for a given CPU. 1587 */ 1588 unsigned int cpufreq_quick_get_max(unsigned int cpu) 1589 { 1590 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1591 unsigned int ret_freq = 0; 1592 1593 if (policy) { 1594 ret_freq = policy->max; 1595 cpufreq_cpu_put(policy); 1596 } 1597 1598 return ret_freq; 1599 } 1600 EXPORT_SYMBOL(cpufreq_quick_get_max); 1601 1602 static unsigned int __cpufreq_get(struct cpufreq_policy *policy) 1603 { 1604 unsigned int ret_freq = 0; 1605 1606 if (unlikely(policy_is_inactive(policy))) 1607 return ret_freq; 1608 1609 ret_freq = cpufreq_driver->get(policy->cpu); 1610 1611 /* 1612 * If fast frequency switching is used with the given policy, the check 1613 * against policy->cur is pointless, so skip it in that case too. 1614 */ 1615 if (policy->fast_switch_enabled) 1616 return ret_freq; 1617 1618 if (ret_freq && policy->cur && 1619 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) { 1620 /* verify no discrepancy between actual and 1621 saved value exists */ 1622 if (unlikely(ret_freq != policy->cur)) { 1623 cpufreq_out_of_sync(policy, ret_freq); 1624 schedule_work(&policy->update); 1625 } 1626 } 1627 1628 return ret_freq; 1629 } 1630 1631 /** 1632 * cpufreq_get - get the current CPU frequency (in kHz) 1633 * @cpu: CPU number 1634 * 1635 * Get the CPU current (static) CPU frequency 1636 */ 1637 unsigned int cpufreq_get(unsigned int cpu) 1638 { 1639 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1640 unsigned int ret_freq = 0; 1641 1642 if (policy) { 1643 down_read(&policy->rwsem); 1644 if (cpufreq_driver->get) 1645 ret_freq = __cpufreq_get(policy); 1646 up_read(&policy->rwsem); 1647 1648 cpufreq_cpu_put(policy); 1649 } 1650 1651 return ret_freq; 1652 } 1653 EXPORT_SYMBOL(cpufreq_get); 1654 1655 static unsigned int cpufreq_update_current_freq(struct cpufreq_policy *policy) 1656 { 1657 unsigned int new_freq; 1658 1659 new_freq = cpufreq_driver->get(policy->cpu); 1660 if (!new_freq) 1661 return 0; 1662 1663 if (!policy->cur) { 1664 pr_debug("cpufreq: Driver did not initialize current freq\n"); 1665 policy->cur = new_freq; 1666 } else if (policy->cur != new_freq && has_target()) { 1667 cpufreq_out_of_sync(policy, new_freq); 1668 } 1669 1670 return new_freq; 1671 } 1672 1673 static struct subsys_interface cpufreq_interface = { 1674 .name = "cpufreq", 1675 .subsys = &cpu_subsys, 1676 .add_dev = cpufreq_add_dev, 1677 .remove_dev = cpufreq_remove_dev, 1678 }; 1679 1680 /* 1681 * In case platform wants some specific frequency to be configured 1682 * during suspend.. 1683 */ 1684 int cpufreq_generic_suspend(struct cpufreq_policy *policy) 1685 { 1686 int ret; 1687 1688 if (!policy->suspend_freq) { 1689 pr_debug("%s: suspend_freq not defined\n", __func__); 1690 return 0; 1691 } 1692 1693 pr_debug("%s: Setting suspend-freq: %u\n", __func__, 1694 policy->suspend_freq); 1695 1696 ret = __cpufreq_driver_target(policy, policy->suspend_freq, 1697 CPUFREQ_RELATION_H); 1698 if (ret) 1699 pr_err("%s: unable to set suspend-freq: %u. err: %d\n", 1700 __func__, policy->suspend_freq, ret); 1701 1702 return ret; 1703 } 1704 EXPORT_SYMBOL(cpufreq_generic_suspend); 1705 1706 /** 1707 * cpufreq_suspend() - Suspend CPUFreq governors 1708 * 1709 * Called during system wide Suspend/Hibernate cycles for suspending governors 1710 * as some platforms can't change frequency after this point in suspend cycle. 1711 * Because some of the devices (like: i2c, regulators, etc) they use for 1712 * changing frequency are suspended quickly after this point. 1713 */ 1714 void cpufreq_suspend(void) 1715 { 1716 struct cpufreq_policy *policy; 1717 1718 if (!cpufreq_driver) 1719 return; 1720 1721 if (!has_target() && !cpufreq_driver->suspend) 1722 goto suspend; 1723 1724 pr_debug("%s: Suspending Governors\n", __func__); 1725 1726 for_each_active_policy(policy) { 1727 if (has_target()) { 1728 down_write(&policy->rwsem); 1729 cpufreq_stop_governor(policy); 1730 up_write(&policy->rwsem); 1731 } 1732 1733 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy)) 1734 pr_err("%s: Failed to suspend driver: %p\n", __func__, 1735 policy); 1736 } 1737 1738 suspend: 1739 cpufreq_suspended = true; 1740 } 1741 1742 /** 1743 * cpufreq_resume() - Resume CPUFreq governors 1744 * 1745 * Called during system wide Suspend/Hibernate cycle for resuming governors that 1746 * are suspended with cpufreq_suspend(). 1747 */ 1748 void cpufreq_resume(void) 1749 { 1750 struct cpufreq_policy *policy; 1751 int ret; 1752 1753 if (!cpufreq_driver) 1754 return; 1755 1756 if (unlikely(!cpufreq_suspended)) 1757 return; 1758 1759 cpufreq_suspended = false; 1760 1761 if (!has_target() && !cpufreq_driver->resume) 1762 return; 1763 1764 pr_debug("%s: Resuming Governors\n", __func__); 1765 1766 for_each_active_policy(policy) { 1767 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) { 1768 pr_err("%s: Failed to resume driver: %p\n", __func__, 1769 policy); 1770 } else if (has_target()) { 1771 down_write(&policy->rwsem); 1772 ret = cpufreq_start_governor(policy); 1773 up_write(&policy->rwsem); 1774 1775 if (ret) 1776 pr_err("%s: Failed to start governor for policy: %p\n", 1777 __func__, policy); 1778 } 1779 } 1780 } 1781 1782 /** 1783 * cpufreq_get_current_driver - return current driver's name 1784 * 1785 * Return the name string of the currently loaded cpufreq driver 1786 * or NULL, if none. 1787 */ 1788 const char *cpufreq_get_current_driver(void) 1789 { 1790 if (cpufreq_driver) 1791 return cpufreq_driver->name; 1792 1793 return NULL; 1794 } 1795 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver); 1796 1797 /** 1798 * cpufreq_get_driver_data - return current driver data 1799 * 1800 * Return the private data of the currently loaded cpufreq 1801 * driver, or NULL if no cpufreq driver is loaded. 1802 */ 1803 void *cpufreq_get_driver_data(void) 1804 { 1805 if (cpufreq_driver) 1806 return cpufreq_driver->driver_data; 1807 1808 return NULL; 1809 } 1810 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data); 1811 1812 /********************************************************************* 1813 * NOTIFIER LISTS INTERFACE * 1814 *********************************************************************/ 1815 1816 /** 1817 * cpufreq_register_notifier - register a driver with cpufreq 1818 * @nb: notifier function to register 1819 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER 1820 * 1821 * Add a driver to one of two lists: either a list of drivers that 1822 * are notified about clock rate changes (once before and once after 1823 * the transition), or a list of drivers that are notified about 1824 * changes in cpufreq policy. 1825 * 1826 * This function may sleep, and has the same return conditions as 1827 * blocking_notifier_chain_register. 1828 */ 1829 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list) 1830 { 1831 int ret; 1832 1833 if (cpufreq_disabled()) 1834 return -EINVAL; 1835 1836 switch (list) { 1837 case CPUFREQ_TRANSITION_NOTIFIER: 1838 mutex_lock(&cpufreq_fast_switch_lock); 1839 1840 if (cpufreq_fast_switch_count > 0) { 1841 mutex_unlock(&cpufreq_fast_switch_lock); 1842 return -EBUSY; 1843 } 1844 ret = srcu_notifier_chain_register( 1845 &cpufreq_transition_notifier_list, nb); 1846 if (!ret) 1847 cpufreq_fast_switch_count--; 1848 1849 mutex_unlock(&cpufreq_fast_switch_lock); 1850 break; 1851 case CPUFREQ_POLICY_NOTIFIER: 1852 ret = blocking_notifier_chain_register( 1853 &cpufreq_policy_notifier_list, nb); 1854 break; 1855 default: 1856 ret = -EINVAL; 1857 } 1858 1859 return ret; 1860 } 1861 EXPORT_SYMBOL(cpufreq_register_notifier); 1862 1863 /** 1864 * cpufreq_unregister_notifier - unregister a driver with cpufreq 1865 * @nb: notifier block to be unregistered 1866 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER 1867 * 1868 * Remove a driver from the CPU frequency notifier list. 1869 * 1870 * This function may sleep, and has the same return conditions as 1871 * blocking_notifier_chain_unregister. 1872 */ 1873 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list) 1874 { 1875 int ret; 1876 1877 if (cpufreq_disabled()) 1878 return -EINVAL; 1879 1880 switch (list) { 1881 case CPUFREQ_TRANSITION_NOTIFIER: 1882 mutex_lock(&cpufreq_fast_switch_lock); 1883 1884 ret = srcu_notifier_chain_unregister( 1885 &cpufreq_transition_notifier_list, nb); 1886 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0)) 1887 cpufreq_fast_switch_count++; 1888 1889 mutex_unlock(&cpufreq_fast_switch_lock); 1890 break; 1891 case CPUFREQ_POLICY_NOTIFIER: 1892 ret = blocking_notifier_chain_unregister( 1893 &cpufreq_policy_notifier_list, nb); 1894 break; 1895 default: 1896 ret = -EINVAL; 1897 } 1898 1899 return ret; 1900 } 1901 EXPORT_SYMBOL(cpufreq_unregister_notifier); 1902 1903 1904 /********************************************************************* 1905 * GOVERNORS * 1906 *********************************************************************/ 1907 1908 /** 1909 * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch. 1910 * @policy: cpufreq policy to switch the frequency for. 1911 * @target_freq: New frequency to set (may be approximate). 1912 * 1913 * Carry out a fast frequency switch without sleeping. 1914 * 1915 * The driver's ->fast_switch() callback invoked by this function must be 1916 * suitable for being called from within RCU-sched read-side critical sections 1917 * and it is expected to select the minimum available frequency greater than or 1918 * equal to @target_freq (CPUFREQ_RELATION_L). 1919 * 1920 * This function must not be called if policy->fast_switch_enabled is unset. 1921 * 1922 * Governors calling this function must guarantee that it will never be invoked 1923 * twice in parallel for the same policy and that it will never be called in 1924 * parallel with either ->target() or ->target_index() for the same policy. 1925 * 1926 * Returns the actual frequency set for the CPU. 1927 * 1928 * If 0 is returned by the driver's ->fast_switch() callback to indicate an 1929 * error condition, the hardware configuration must be preserved. 1930 */ 1931 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, 1932 unsigned int target_freq) 1933 { 1934 target_freq = clamp_val(target_freq, policy->min, policy->max); 1935 1936 return cpufreq_driver->fast_switch(policy, target_freq); 1937 } 1938 EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); 1939 1940 /* Must set freqs->new to intermediate frequency */ 1941 static int __target_intermediate(struct cpufreq_policy *policy, 1942 struct cpufreq_freqs *freqs, int index) 1943 { 1944 int ret; 1945 1946 freqs->new = cpufreq_driver->get_intermediate(policy, index); 1947 1948 /* We don't need to switch to intermediate freq */ 1949 if (!freqs->new) 1950 return 0; 1951 1952 pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n", 1953 __func__, policy->cpu, freqs->old, freqs->new); 1954 1955 cpufreq_freq_transition_begin(policy, freqs); 1956 ret = cpufreq_driver->target_intermediate(policy, index); 1957 cpufreq_freq_transition_end(policy, freqs, ret); 1958 1959 if (ret) 1960 pr_err("%s: Failed to change to intermediate frequency: %d\n", 1961 __func__, ret); 1962 1963 return ret; 1964 } 1965 1966 static int __target_index(struct cpufreq_policy *policy, int index) 1967 { 1968 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0}; 1969 unsigned int intermediate_freq = 0; 1970 unsigned int newfreq = policy->freq_table[index].frequency; 1971 int retval = -EINVAL; 1972 bool notify; 1973 1974 if (newfreq == policy->cur) 1975 return 0; 1976 1977 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION); 1978 if (notify) { 1979 /* Handle switching to intermediate frequency */ 1980 if (cpufreq_driver->get_intermediate) { 1981 retval = __target_intermediate(policy, &freqs, index); 1982 if (retval) 1983 return retval; 1984 1985 intermediate_freq = freqs.new; 1986 /* Set old freq to intermediate */ 1987 if (intermediate_freq) 1988 freqs.old = freqs.new; 1989 } 1990 1991 freqs.new = newfreq; 1992 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n", 1993 __func__, policy->cpu, freqs.old, freqs.new); 1994 1995 cpufreq_freq_transition_begin(policy, &freqs); 1996 } 1997 1998 retval = cpufreq_driver->target_index(policy, index); 1999 if (retval) 2000 pr_err("%s: Failed to change cpu frequency: %d\n", __func__, 2001 retval); 2002 2003 if (notify) { 2004 cpufreq_freq_transition_end(policy, &freqs, retval); 2005 2006 /* 2007 * Failed after setting to intermediate freq? Driver should have 2008 * reverted back to initial frequency and so should we. Check 2009 * here for intermediate_freq instead of get_intermediate, in 2010 * case we haven't switched to intermediate freq at all. 2011 */ 2012 if (unlikely(retval && intermediate_freq)) { 2013 freqs.old = intermediate_freq; 2014 freqs.new = policy->restore_freq; 2015 cpufreq_freq_transition_begin(policy, &freqs); 2016 cpufreq_freq_transition_end(policy, &freqs, 0); 2017 } 2018 } 2019 2020 return retval; 2021 } 2022 2023 int __cpufreq_driver_target(struct cpufreq_policy *policy, 2024 unsigned int target_freq, 2025 unsigned int relation) 2026 { 2027 unsigned int old_target_freq = target_freq; 2028 int index; 2029 2030 if (cpufreq_disabled()) 2031 return -ENODEV; 2032 2033 /* Make sure that target_freq is within supported range */ 2034 target_freq = clamp_val(target_freq, policy->min, policy->max); 2035 2036 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n", 2037 policy->cpu, target_freq, relation, old_target_freq); 2038 2039 /* 2040 * This might look like a redundant call as we are checking it again 2041 * after finding index. But it is left intentionally for cases where 2042 * exactly same freq is called again and so we can save on few function 2043 * calls. 2044 */ 2045 if (target_freq == policy->cur) 2046 return 0; 2047 2048 /* Save last value to restore later on errors */ 2049 policy->restore_freq = policy->cur; 2050 2051 if (cpufreq_driver->target) 2052 return cpufreq_driver->target(policy, target_freq, relation); 2053 2054 if (!cpufreq_driver->target_index) 2055 return -EINVAL; 2056 2057 index = cpufreq_frequency_table_target(policy, target_freq, relation); 2058 2059 return __target_index(policy, index); 2060 } 2061 EXPORT_SYMBOL_GPL(__cpufreq_driver_target); 2062 2063 int cpufreq_driver_target(struct cpufreq_policy *policy, 2064 unsigned int target_freq, 2065 unsigned int relation) 2066 { 2067 int ret = -EINVAL; 2068 2069 down_write(&policy->rwsem); 2070 2071 ret = __cpufreq_driver_target(policy, target_freq, relation); 2072 2073 up_write(&policy->rwsem); 2074 2075 return ret; 2076 } 2077 EXPORT_SYMBOL_GPL(cpufreq_driver_target); 2078 2079 __weak struct cpufreq_governor *cpufreq_fallback_governor(void) 2080 { 2081 return NULL; 2082 } 2083 2084 static int cpufreq_init_governor(struct cpufreq_policy *policy) 2085 { 2086 int ret; 2087 2088 /* Don't start any governor operations if we are entering suspend */ 2089 if (cpufreq_suspended) 2090 return 0; 2091 /* 2092 * Governor might not be initiated here if ACPI _PPC changed 2093 * notification happened, so check it. 2094 */ 2095 if (!policy->governor) 2096 return -EINVAL; 2097 2098 /* Platform doesn't want dynamic frequency switching ? */ 2099 if (policy->governor->dynamic_switching && 2100 cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) { 2101 struct cpufreq_governor *gov = cpufreq_fallback_governor(); 2102 2103 if (gov) { 2104 pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n", 2105 policy->governor->name, gov->name); 2106 policy->governor = gov; 2107 } else { 2108 return -EINVAL; 2109 } 2110 } 2111 2112 if (!try_module_get(policy->governor->owner)) 2113 return -EINVAL; 2114 2115 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2116 2117 if (policy->governor->init) { 2118 ret = policy->governor->init(policy); 2119 if (ret) { 2120 module_put(policy->governor->owner); 2121 return ret; 2122 } 2123 } 2124 2125 return 0; 2126 } 2127 2128 static void cpufreq_exit_governor(struct cpufreq_policy *policy) 2129 { 2130 if (cpufreq_suspended || !policy->governor) 2131 return; 2132 2133 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2134 2135 if (policy->governor->exit) 2136 policy->governor->exit(policy); 2137 2138 module_put(policy->governor->owner); 2139 } 2140 2141 static int cpufreq_start_governor(struct cpufreq_policy *policy) 2142 { 2143 int ret; 2144 2145 if (cpufreq_suspended) 2146 return 0; 2147 2148 if (!policy->governor) 2149 return -EINVAL; 2150 2151 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2152 2153 if (cpufreq_driver->get && !cpufreq_driver->setpolicy) 2154 cpufreq_update_current_freq(policy); 2155 2156 if (policy->governor->start) { 2157 ret = policy->governor->start(policy); 2158 if (ret) 2159 return ret; 2160 } 2161 2162 if (policy->governor->limits) 2163 policy->governor->limits(policy); 2164 2165 return 0; 2166 } 2167 2168 static void cpufreq_stop_governor(struct cpufreq_policy *policy) 2169 { 2170 if (cpufreq_suspended || !policy->governor) 2171 return; 2172 2173 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2174 2175 if (policy->governor->stop) 2176 policy->governor->stop(policy); 2177 } 2178 2179 static void cpufreq_governor_limits(struct cpufreq_policy *policy) 2180 { 2181 if (cpufreq_suspended || !policy->governor) 2182 return; 2183 2184 pr_debug("%s: for CPU %u\n", __func__, policy->cpu); 2185 2186 if (policy->governor->limits) 2187 policy->governor->limits(policy); 2188 } 2189 2190 int cpufreq_register_governor(struct cpufreq_governor *governor) 2191 { 2192 int err; 2193 2194 if (!governor) 2195 return -EINVAL; 2196 2197 if (cpufreq_disabled()) 2198 return -ENODEV; 2199 2200 mutex_lock(&cpufreq_governor_mutex); 2201 2202 err = -EBUSY; 2203 if (!find_governor(governor->name)) { 2204 err = 0; 2205 list_add(&governor->governor_list, &cpufreq_governor_list); 2206 } 2207 2208 mutex_unlock(&cpufreq_governor_mutex); 2209 return err; 2210 } 2211 EXPORT_SYMBOL_GPL(cpufreq_register_governor); 2212 2213 void cpufreq_unregister_governor(struct cpufreq_governor *governor) 2214 { 2215 struct cpufreq_policy *policy; 2216 unsigned long flags; 2217 2218 if (!governor) 2219 return; 2220 2221 if (cpufreq_disabled()) 2222 return; 2223 2224 /* clear last_governor for all inactive policies */ 2225 read_lock_irqsave(&cpufreq_driver_lock, flags); 2226 for_each_inactive_policy(policy) { 2227 if (!strcmp(policy->last_governor, governor->name)) { 2228 policy->governor = NULL; 2229 strcpy(policy->last_governor, "\0"); 2230 } 2231 } 2232 read_unlock_irqrestore(&cpufreq_driver_lock, flags); 2233 2234 mutex_lock(&cpufreq_governor_mutex); 2235 list_del(&governor->governor_list); 2236 mutex_unlock(&cpufreq_governor_mutex); 2237 } 2238 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor); 2239 2240 2241 /********************************************************************* 2242 * POLICY INTERFACE * 2243 *********************************************************************/ 2244 2245 /** 2246 * cpufreq_get_policy - get the current cpufreq_policy 2247 * @policy: struct cpufreq_policy into which the current cpufreq_policy 2248 * is written 2249 * 2250 * Reads the current cpufreq policy. 2251 */ 2252 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu) 2253 { 2254 struct cpufreq_policy *cpu_policy; 2255 if (!policy) 2256 return -EINVAL; 2257 2258 cpu_policy = cpufreq_cpu_get(cpu); 2259 if (!cpu_policy) 2260 return -EINVAL; 2261 2262 memcpy(policy, cpu_policy, sizeof(*policy)); 2263 2264 cpufreq_cpu_put(cpu_policy); 2265 return 0; 2266 } 2267 EXPORT_SYMBOL(cpufreq_get_policy); 2268 2269 /** 2270 * cpufreq_set_policy - Modify cpufreq policy parameters. 2271 * @policy: Policy object to modify. 2272 * @new_policy: New policy data. 2273 * 2274 * Pass @new_policy to the cpufreq driver's ->verify() callback, run the 2275 * installed policy notifiers for it with the CPUFREQ_ADJUST value, pass it to 2276 * the driver's ->verify() callback again and run the notifiers for it again 2277 * with the CPUFREQ_NOTIFY value. Next, copy the min and max parameters 2278 * of @new_policy to @policy and either invoke the driver's ->setpolicy() 2279 * callback (if present) or carry out a governor update for @policy. That is, 2280 * run the current governor's ->limits() callback (if the governor field in 2281 * @new_policy points to the same object as the one in @policy) or replace the 2282 * governor for @policy with the new one stored in @new_policy. 2283 * 2284 * The cpuinfo part of @policy is not updated by this function. 2285 */ 2286 int cpufreq_set_policy(struct cpufreq_policy *policy, 2287 struct cpufreq_policy *new_policy) 2288 { 2289 struct cpufreq_governor *old_gov; 2290 int ret; 2291 2292 pr_debug("setting new policy for CPU %u: %u - %u kHz\n", 2293 new_policy->cpu, new_policy->min, new_policy->max); 2294 2295 memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo)); 2296 2297 /* 2298 * This check works well when we store new min/max freq attributes, 2299 * because new_policy is a copy of policy with one field updated. 2300 */ 2301 if (new_policy->min > new_policy->max) 2302 return -EINVAL; 2303 2304 /* verify the cpu speed can be set within this limit */ 2305 ret = cpufreq_driver->verify(new_policy); 2306 if (ret) 2307 return ret; 2308 2309 /* adjust if necessary - all reasons */ 2310 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 2311 CPUFREQ_ADJUST, new_policy); 2312 2313 /* 2314 * verify the cpu speed can be set within this limit, which might be 2315 * different to the first one 2316 */ 2317 ret = cpufreq_driver->verify(new_policy); 2318 if (ret) 2319 return ret; 2320 2321 /* notification of the new policy */ 2322 blocking_notifier_call_chain(&cpufreq_policy_notifier_list, 2323 CPUFREQ_NOTIFY, new_policy); 2324 2325 policy->min = new_policy->min; 2326 policy->max = new_policy->max; 2327 trace_cpu_frequency_limits(policy); 2328 2329 policy->cached_target_freq = UINT_MAX; 2330 2331 pr_debug("new min and max freqs are %u - %u kHz\n", 2332 policy->min, policy->max); 2333 2334 if (cpufreq_driver->setpolicy) { 2335 policy->policy = new_policy->policy; 2336 pr_debug("setting range\n"); 2337 return cpufreq_driver->setpolicy(policy); 2338 } 2339 2340 if (new_policy->governor == policy->governor) { 2341 pr_debug("governor limits update\n"); 2342 cpufreq_governor_limits(policy); 2343 return 0; 2344 } 2345 2346 pr_debug("governor switch\n"); 2347 2348 /* save old, working values */ 2349 old_gov = policy->governor; 2350 /* end old governor */ 2351 if (old_gov) { 2352 cpufreq_stop_governor(policy); 2353 cpufreq_exit_governor(policy); 2354 } 2355 2356 /* start new governor */ 2357 policy->governor = new_policy->governor; 2358 ret = cpufreq_init_governor(policy); 2359 if (!ret) { 2360 ret = cpufreq_start_governor(policy); 2361 if (!ret) { 2362 pr_debug("governor change\n"); 2363 sched_cpufreq_governor_change(policy, old_gov); 2364 return 0; 2365 } 2366 cpufreq_exit_governor(policy); 2367 } 2368 2369 /* new governor failed, so re-start old one */ 2370 pr_debug("starting governor %s failed\n", policy->governor->name); 2371 if (old_gov) { 2372 policy->governor = old_gov; 2373 if (cpufreq_init_governor(policy)) 2374 policy->governor = NULL; 2375 else 2376 cpufreq_start_governor(policy); 2377 } 2378 2379 return ret; 2380 } 2381 2382 /** 2383 * cpufreq_update_policy - Re-evaluate an existing cpufreq policy. 2384 * @cpu: CPU to re-evaluate the policy for. 2385 * 2386 * Update the current frequency for the cpufreq policy of @cpu and use 2387 * cpufreq_set_policy() to re-apply the min and max limits saved in the 2388 * user_policy sub-structure of that policy, which triggers the evaluation 2389 * of policy notifiers and the cpufreq driver's ->verify() callback for the 2390 * policy in question, among other things. 2391 */ 2392 void cpufreq_update_policy(unsigned int cpu) 2393 { 2394 struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu); 2395 struct cpufreq_policy new_policy; 2396 2397 if (!policy) 2398 return; 2399 2400 /* 2401 * BIOS might change freq behind our back 2402 * -> ask driver for current freq and notify governors about a change 2403 */ 2404 if (cpufreq_driver->get && !cpufreq_driver->setpolicy && 2405 (cpufreq_suspended || WARN_ON(!cpufreq_update_current_freq(policy)))) 2406 goto unlock; 2407 2408 pr_debug("updating policy for CPU %u\n", cpu); 2409 memcpy(&new_policy, policy, sizeof(*policy)); 2410 new_policy.min = policy->user_policy.min; 2411 new_policy.max = policy->user_policy.max; 2412 2413 cpufreq_set_policy(policy, &new_policy); 2414 2415 unlock: 2416 cpufreq_cpu_release(policy); 2417 } 2418 EXPORT_SYMBOL(cpufreq_update_policy); 2419 2420 /** 2421 * cpufreq_update_limits - Update policy limits for a given CPU. 2422 * @cpu: CPU to update the policy limits for. 2423 * 2424 * Invoke the driver's ->update_limits callback if present or call 2425 * cpufreq_update_policy() for @cpu. 2426 */ 2427 void cpufreq_update_limits(unsigned int cpu) 2428 { 2429 if (cpufreq_driver->update_limits) 2430 cpufreq_driver->update_limits(cpu); 2431 else 2432 cpufreq_update_policy(cpu); 2433 } 2434 EXPORT_SYMBOL_GPL(cpufreq_update_limits); 2435 2436 /********************************************************************* 2437 * BOOST * 2438 *********************************************************************/ 2439 static int cpufreq_boost_set_sw(int state) 2440 { 2441 struct cpufreq_policy *policy; 2442 int ret = -EINVAL; 2443 2444 for_each_active_policy(policy) { 2445 if (!policy->freq_table) 2446 continue; 2447 2448 ret = cpufreq_frequency_table_cpuinfo(policy, 2449 policy->freq_table); 2450 if (ret) { 2451 pr_err("%s: Policy frequency update failed\n", 2452 __func__); 2453 break; 2454 } 2455 2456 down_write(&policy->rwsem); 2457 policy->user_policy.max = policy->max; 2458 cpufreq_governor_limits(policy); 2459 up_write(&policy->rwsem); 2460 } 2461 2462 return ret; 2463 } 2464 2465 int cpufreq_boost_trigger_state(int state) 2466 { 2467 unsigned long flags; 2468 int ret = 0; 2469 2470 if (cpufreq_driver->boost_enabled == state) 2471 return 0; 2472 2473 write_lock_irqsave(&cpufreq_driver_lock, flags); 2474 cpufreq_driver->boost_enabled = state; 2475 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2476 2477 ret = cpufreq_driver->set_boost(state); 2478 if (ret) { 2479 write_lock_irqsave(&cpufreq_driver_lock, flags); 2480 cpufreq_driver->boost_enabled = !state; 2481 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2482 2483 pr_err("%s: Cannot %s BOOST\n", 2484 __func__, state ? "enable" : "disable"); 2485 } 2486 2487 return ret; 2488 } 2489 2490 static bool cpufreq_boost_supported(void) 2491 { 2492 return cpufreq_driver->set_boost; 2493 } 2494 2495 static int create_boost_sysfs_file(void) 2496 { 2497 int ret; 2498 2499 ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr); 2500 if (ret) 2501 pr_err("%s: cannot register global BOOST sysfs file\n", 2502 __func__); 2503 2504 return ret; 2505 } 2506 2507 static void remove_boost_sysfs_file(void) 2508 { 2509 if (cpufreq_boost_supported()) 2510 sysfs_remove_file(cpufreq_global_kobject, &boost.attr); 2511 } 2512 2513 int cpufreq_enable_boost_support(void) 2514 { 2515 if (!cpufreq_driver) 2516 return -EINVAL; 2517 2518 if (cpufreq_boost_supported()) 2519 return 0; 2520 2521 cpufreq_driver->set_boost = cpufreq_boost_set_sw; 2522 2523 /* This will get removed on driver unregister */ 2524 return create_boost_sysfs_file(); 2525 } 2526 EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support); 2527 2528 int cpufreq_boost_enabled(void) 2529 { 2530 return cpufreq_driver->boost_enabled; 2531 } 2532 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled); 2533 2534 /********************************************************************* 2535 * REGISTER / UNREGISTER CPUFREQ DRIVER * 2536 *********************************************************************/ 2537 static enum cpuhp_state hp_online; 2538 2539 static int cpuhp_cpufreq_online(unsigned int cpu) 2540 { 2541 cpufreq_online(cpu); 2542 2543 return 0; 2544 } 2545 2546 static int cpuhp_cpufreq_offline(unsigned int cpu) 2547 { 2548 cpufreq_offline(cpu); 2549 2550 return 0; 2551 } 2552 2553 /** 2554 * cpufreq_register_driver - register a CPU Frequency driver 2555 * @driver_data: A struct cpufreq_driver containing the values# 2556 * submitted by the CPU Frequency driver. 2557 * 2558 * Registers a CPU Frequency driver to this core code. This code 2559 * returns zero on success, -EEXIST when another driver got here first 2560 * (and isn't unregistered in the meantime). 2561 * 2562 */ 2563 int cpufreq_register_driver(struct cpufreq_driver *driver_data) 2564 { 2565 unsigned long flags; 2566 int ret; 2567 2568 if (cpufreq_disabled()) 2569 return -ENODEV; 2570 2571 if (!driver_data || !driver_data->verify || !driver_data->init || 2572 !(driver_data->setpolicy || driver_data->target_index || 2573 driver_data->target) || 2574 (driver_data->setpolicy && (driver_data->target_index || 2575 driver_data->target)) || 2576 (!driver_data->get_intermediate != !driver_data->target_intermediate) || 2577 (!driver_data->online != !driver_data->offline)) 2578 return -EINVAL; 2579 2580 pr_debug("trying to register driver %s\n", driver_data->name); 2581 2582 /* Protect against concurrent CPU online/offline. */ 2583 cpus_read_lock(); 2584 2585 write_lock_irqsave(&cpufreq_driver_lock, flags); 2586 if (cpufreq_driver) { 2587 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2588 ret = -EEXIST; 2589 goto out; 2590 } 2591 cpufreq_driver = driver_data; 2592 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2593 2594 if (driver_data->setpolicy) 2595 driver_data->flags |= CPUFREQ_CONST_LOOPS; 2596 2597 if (cpufreq_boost_supported()) { 2598 ret = create_boost_sysfs_file(); 2599 if (ret) 2600 goto err_null_driver; 2601 } 2602 2603 ret = subsys_interface_register(&cpufreq_interface); 2604 if (ret) 2605 goto err_boost_unreg; 2606 2607 if (!(cpufreq_driver->flags & CPUFREQ_STICKY) && 2608 list_empty(&cpufreq_policy_list)) { 2609 /* if all ->init() calls failed, unregister */ 2610 ret = -ENODEV; 2611 pr_debug("%s: No CPU initialized for driver %s\n", __func__, 2612 driver_data->name); 2613 goto err_if_unreg; 2614 } 2615 2616 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, 2617 "cpufreq:online", 2618 cpuhp_cpufreq_online, 2619 cpuhp_cpufreq_offline); 2620 if (ret < 0) 2621 goto err_if_unreg; 2622 hp_online = ret; 2623 ret = 0; 2624 2625 pr_debug("driver %s up and running\n", driver_data->name); 2626 goto out; 2627 2628 err_if_unreg: 2629 subsys_interface_unregister(&cpufreq_interface); 2630 err_boost_unreg: 2631 remove_boost_sysfs_file(); 2632 err_null_driver: 2633 write_lock_irqsave(&cpufreq_driver_lock, flags); 2634 cpufreq_driver = NULL; 2635 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2636 out: 2637 cpus_read_unlock(); 2638 return ret; 2639 } 2640 EXPORT_SYMBOL_GPL(cpufreq_register_driver); 2641 2642 /** 2643 * cpufreq_unregister_driver - unregister the current CPUFreq driver 2644 * 2645 * Unregister the current CPUFreq driver. Only call this if you have 2646 * the right to do so, i.e. if you have succeeded in initialising before! 2647 * Returns zero if successful, and -EINVAL if the cpufreq_driver is 2648 * currently not initialised. 2649 */ 2650 int cpufreq_unregister_driver(struct cpufreq_driver *driver) 2651 { 2652 unsigned long flags; 2653 2654 if (!cpufreq_driver || (driver != cpufreq_driver)) 2655 return -EINVAL; 2656 2657 pr_debug("unregistering driver %s\n", driver->name); 2658 2659 /* Protect against concurrent cpu hotplug */ 2660 cpus_read_lock(); 2661 subsys_interface_unregister(&cpufreq_interface); 2662 remove_boost_sysfs_file(); 2663 cpuhp_remove_state_nocalls_cpuslocked(hp_online); 2664 2665 write_lock_irqsave(&cpufreq_driver_lock, flags); 2666 2667 cpufreq_driver = NULL; 2668 2669 write_unlock_irqrestore(&cpufreq_driver_lock, flags); 2670 cpus_read_unlock(); 2671 2672 return 0; 2673 } 2674 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver); 2675 2676 /* 2677 * Stop cpufreq at shutdown to make sure it isn't holding any locks 2678 * or mutexes when secondary CPUs are halted. 2679 */ 2680 static struct syscore_ops cpufreq_syscore_ops = { 2681 .shutdown = cpufreq_suspend, 2682 }; 2683 2684 struct kobject *cpufreq_global_kobject; 2685 EXPORT_SYMBOL(cpufreq_global_kobject); 2686 2687 static int __init cpufreq_core_init(void) 2688 { 2689 if (cpufreq_disabled()) 2690 return -ENODEV; 2691 2692 cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj); 2693 BUG_ON(!cpufreq_global_kobject); 2694 2695 register_syscore_ops(&cpufreq_syscore_ops); 2696 2697 return 0; 2698 } 2699 module_param(off, int, 0444); 2700 core_initcall(cpufreq_core_init); 2701