1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * linux/drivers/cpufreq/cpufreq.c
4 *
5 * Copyright (C) 2001 Russell King
6 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
7 * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
8 *
9 * Oct 2005 - Ashok Raj <ashok.raj@intel.com>
10 * Added handling for CPU hotplug
11 * Feb 2006 - Jacob Shin <jacob.shin@amd.com>
12 * Fix handling for CPU hotplug -- affected CPUs
13 */
14
15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16
17 #include <linux/cpu.h>
18 #include <linux/cpufreq.h>
19 #include <linux/cpu_cooling.h>
20 #include <linux/delay.h>
21 #include <linux/device.h>
22 #include <linux/init.h>
23 #include <linux/kernel_stat.h>
24 #include <linux/module.h>
25 #include <linux/mutex.h>
26 #include <linux/pm_qos.h>
27 #include <linux/slab.h>
28 #include <linux/suspend.h>
29 #include <linux/syscore_ops.h>
30 #include <linux/tick.h>
31 #include <linux/units.h>
32 #include <trace/events/power.h>
33
34 static LIST_HEAD(cpufreq_policy_list);
35
36 /* Macros to iterate over CPU policies */
37 #define for_each_suitable_policy(__policy, __active) \
38 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
39 if ((__active) == !policy_is_inactive(__policy))
40
41 #define for_each_active_policy(__policy) \
42 for_each_suitable_policy(__policy, true)
43 #define for_each_inactive_policy(__policy) \
44 for_each_suitable_policy(__policy, false)
45
46 /* Iterate over governors */
47 static LIST_HEAD(cpufreq_governor_list);
48 #define for_each_governor(__governor) \
49 list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
50
51 static char default_governor[CPUFREQ_NAME_LEN];
52
53 /*
54 * The "cpufreq driver" - the arch- or hardware-dependent low
55 * level driver of CPUFreq support, and its spinlock. This lock
56 * also protects the cpufreq_cpu_data array.
57 */
58 static struct cpufreq_driver *cpufreq_driver;
59 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
60 static DEFINE_RWLOCK(cpufreq_driver_lock);
61
62 static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance);
cpufreq_supports_freq_invariance(void)63 bool cpufreq_supports_freq_invariance(void)
64 {
65 return static_branch_likely(&cpufreq_freq_invariance);
66 }
67
68 /* Flag to suspend/resume CPUFreq governors */
69 static bool cpufreq_suspended;
70
has_target(void)71 static inline bool has_target(void)
72 {
73 return cpufreq_driver->target_index || cpufreq_driver->target;
74 }
75
has_target_index(void)76 bool has_target_index(void)
77 {
78 return !!cpufreq_driver->target_index;
79 }
80
81 /* internal prototypes */
82 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
83 static int cpufreq_init_governor(struct cpufreq_policy *policy);
84 static void cpufreq_exit_governor(struct cpufreq_policy *policy);
85 static void cpufreq_governor_limits(struct cpufreq_policy *policy);
86 static int cpufreq_set_policy(struct cpufreq_policy *policy,
87 struct cpufreq_governor *new_gov,
88 unsigned int new_pol);
89 static bool cpufreq_boost_supported(void);
90
91 /*
92 * Two notifier lists: the "policy" list is involved in the
93 * validation process for a new CPU frequency policy; the
94 * "transition" list for kernel code that needs to handle
95 * changes to devices when the CPU clock speed changes.
96 * The mutex locks both lists.
97 */
98 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
99 SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
100
101 static int off __read_mostly;
cpufreq_disabled(void)102 static int cpufreq_disabled(void)
103 {
104 return off;
105 }
disable_cpufreq(void)106 void disable_cpufreq(void)
107 {
108 off = 1;
109 }
110 static DEFINE_MUTEX(cpufreq_governor_mutex);
111
have_governor_per_policy(void)112 bool have_governor_per_policy(void)
113 {
114 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
115 }
116 EXPORT_SYMBOL_GPL(have_governor_per_policy);
117
118 static struct kobject *cpufreq_global_kobject;
119
get_governor_parent_kobj(struct cpufreq_policy * policy)120 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
121 {
122 if (have_governor_per_policy())
123 return &policy->kobj;
124 else
125 return cpufreq_global_kobject;
126 }
127 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
128
get_cpu_idle_time_jiffy(unsigned int cpu,u64 * wall)129 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
130 {
131 struct kernel_cpustat kcpustat;
132 u64 cur_wall_time;
133 u64 idle_time;
134 u64 busy_time;
135
136 cur_wall_time = jiffies64_to_nsecs(get_jiffies_64());
137
138 kcpustat_cpu_fetch(&kcpustat, cpu);
139
140 busy_time = kcpustat.cpustat[CPUTIME_USER];
141 busy_time += kcpustat.cpustat[CPUTIME_SYSTEM];
142 busy_time += kcpustat.cpustat[CPUTIME_IRQ];
143 busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ];
144 busy_time += kcpustat.cpustat[CPUTIME_STEAL];
145 busy_time += kcpustat.cpustat[CPUTIME_NICE];
146
147 idle_time = cur_wall_time - busy_time;
148 if (wall)
149 *wall = div_u64(cur_wall_time, NSEC_PER_USEC);
150
151 return div_u64(idle_time, NSEC_PER_USEC);
152 }
153
get_cpu_idle_time(unsigned int cpu,u64 * wall,int io_busy)154 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
155 {
156 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
157
158 if (idle_time == -1ULL)
159 return get_cpu_idle_time_jiffy(cpu, wall);
160 else if (!io_busy)
161 idle_time += get_cpu_iowait_time_us(cpu, wall);
162
163 return idle_time;
164 }
165 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
166
167 /*
168 * This is a generic cpufreq init() routine which can be used by cpufreq
169 * drivers of SMP systems. It will do following:
170 * - validate & show freq table passed
171 * - set policies transition latency
172 * - policy->cpus with all possible CPUs
173 */
cpufreq_generic_init(struct cpufreq_policy * policy,struct cpufreq_frequency_table * table,unsigned int transition_latency)174 void cpufreq_generic_init(struct cpufreq_policy *policy,
175 struct cpufreq_frequency_table *table,
176 unsigned int transition_latency)
177 {
178 policy->freq_table = table;
179 policy->cpuinfo.transition_latency = transition_latency;
180
181 /*
182 * The driver only supports the SMP configuration where all processors
183 * share the clock and voltage and clock.
184 */
185 cpumask_setall(policy->cpus);
186 }
187 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
188
cpufreq_cpu_get_raw(unsigned int cpu)189 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
190 {
191 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
192
193 return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
194 }
195 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
196
cpufreq_generic_get(unsigned int cpu)197 unsigned int cpufreq_generic_get(unsigned int cpu)
198 {
199 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
200
201 if (!policy || IS_ERR(policy->clk)) {
202 pr_err("%s: No %s associated to cpu: %d\n",
203 __func__, policy ? "clk" : "policy", cpu);
204 return 0;
205 }
206
207 return clk_get_rate(policy->clk) / 1000;
208 }
209 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
210
211 /**
212 * cpufreq_cpu_get - Return policy for a CPU and mark it as busy.
213 * @cpu: CPU to find the policy for.
214 *
215 * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment
216 * the kobject reference counter of that policy. Return a valid policy on
217 * success or NULL on failure.
218 *
219 * The policy returned by this function has to be released with the help of
220 * cpufreq_cpu_put() to balance its kobject reference counter properly.
221 */
cpufreq_cpu_get(unsigned int cpu)222 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
223 {
224 struct cpufreq_policy *policy = NULL;
225 unsigned long flags;
226
227 if (WARN_ON(cpu >= nr_cpu_ids))
228 return NULL;
229
230 /* get the cpufreq driver */
231 read_lock_irqsave(&cpufreq_driver_lock, flags);
232
233 if (cpufreq_driver) {
234 /* get the CPU */
235 policy = cpufreq_cpu_get_raw(cpu);
236 if (policy)
237 kobject_get(&policy->kobj);
238 }
239
240 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
241
242 return policy;
243 }
244 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
245
246 /**
247 * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy.
248 * @policy: cpufreq policy returned by cpufreq_cpu_get().
249 */
cpufreq_cpu_put(struct cpufreq_policy * policy)250 void cpufreq_cpu_put(struct cpufreq_policy *policy)
251 {
252 kobject_put(&policy->kobj);
253 }
254 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
255
256 /**
257 * cpufreq_cpu_release - Unlock a policy and decrement its usage counter.
258 * @policy: cpufreq policy returned by cpufreq_cpu_acquire().
259 */
cpufreq_cpu_release(struct cpufreq_policy * policy)260 void cpufreq_cpu_release(struct cpufreq_policy *policy)
261 {
262 if (WARN_ON(!policy))
263 return;
264
265 lockdep_assert_held(&policy->rwsem);
266
267 up_write(&policy->rwsem);
268
269 cpufreq_cpu_put(policy);
270 }
271
272 /**
273 * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it.
274 * @cpu: CPU to find the policy for.
275 *
276 * Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and
277 * if the policy returned by it is not NULL, acquire its rwsem for writing.
278 * Return the policy if it is active or release it and return NULL otherwise.
279 *
280 * The policy returned by this function has to be released with the help of
281 * cpufreq_cpu_release() in order to release its rwsem and balance its usage
282 * counter properly.
283 */
cpufreq_cpu_acquire(unsigned int cpu)284 struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu)
285 {
286 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
287
288 if (!policy)
289 return NULL;
290
291 down_write(&policy->rwsem);
292
293 if (policy_is_inactive(policy)) {
294 cpufreq_cpu_release(policy);
295 return NULL;
296 }
297
298 return policy;
299 }
300
301 /*********************************************************************
302 * EXTERNALLY AFFECTING FREQUENCY CHANGES *
303 *********************************************************************/
304
305 /**
306 * adjust_jiffies - Adjust the system "loops_per_jiffy".
307 * @val: CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
308 * @ci: Frequency change information.
309 *
310 * This function alters the system "loops_per_jiffy" for the clock
311 * speed change. Note that loops_per_jiffy cannot be updated on SMP
312 * systems as each CPU might be scaled differently. So, use the arch
313 * per-CPU loops_per_jiffy value wherever possible.
314 */
adjust_jiffies(unsigned long val,struct cpufreq_freqs * ci)315 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
316 {
317 #ifndef CONFIG_SMP
318 static unsigned long l_p_j_ref;
319 static unsigned int l_p_j_ref_freq;
320
321 if (ci->flags & CPUFREQ_CONST_LOOPS)
322 return;
323
324 if (!l_p_j_ref_freq) {
325 l_p_j_ref = loops_per_jiffy;
326 l_p_j_ref_freq = ci->old;
327 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
328 l_p_j_ref, l_p_j_ref_freq);
329 }
330 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
331 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
332 ci->new);
333 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
334 loops_per_jiffy, ci->new);
335 }
336 #endif
337 }
338
339 /**
340 * cpufreq_notify_transition - Notify frequency transition and adjust jiffies.
341 * @policy: cpufreq policy to enable fast frequency switching for.
342 * @freqs: contain details of the frequency update.
343 * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
344 *
345 * This function calls the transition notifiers and adjust_jiffies().
346 *
347 * It is called twice on all CPU frequency changes that have external effects.
348 */
cpufreq_notify_transition(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs,unsigned int state)349 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
350 struct cpufreq_freqs *freqs,
351 unsigned int state)
352 {
353 int cpu;
354
355 BUG_ON(irqs_disabled());
356
357 if (cpufreq_disabled())
358 return;
359
360 freqs->policy = policy;
361 freqs->flags = cpufreq_driver->flags;
362 pr_debug("notification %u of frequency transition to %u kHz\n",
363 state, freqs->new);
364
365 switch (state) {
366 case CPUFREQ_PRECHANGE:
367 /*
368 * Detect if the driver reported a value as "old frequency"
369 * which is not equal to what the cpufreq core thinks is
370 * "old frequency".
371 */
372 if (policy->cur && policy->cur != freqs->old) {
373 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
374 freqs->old, policy->cur);
375 freqs->old = policy->cur;
376 }
377
378 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
379 CPUFREQ_PRECHANGE, freqs);
380
381 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
382 break;
383
384 case CPUFREQ_POSTCHANGE:
385 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
386 pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
387 cpumask_pr_args(policy->cpus));
388
389 for_each_cpu(cpu, policy->cpus)
390 trace_cpu_frequency(freqs->new, cpu);
391
392 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
393 CPUFREQ_POSTCHANGE, freqs);
394
395 cpufreq_stats_record_transition(policy, freqs->new);
396 policy->cur = freqs->new;
397 }
398 }
399
400 /* Do post notifications when there are chances that transition has failed */
cpufreq_notify_post_transition(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs,int transition_failed)401 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
402 struct cpufreq_freqs *freqs, int transition_failed)
403 {
404 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
405 if (!transition_failed)
406 return;
407
408 swap(freqs->old, freqs->new);
409 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
410 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
411 }
412
cpufreq_freq_transition_begin(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs)413 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
414 struct cpufreq_freqs *freqs)
415 {
416
417 /*
418 * Catch double invocations of _begin() which lead to self-deadlock.
419 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
420 * doesn't invoke _begin() on their behalf, and hence the chances of
421 * double invocations are very low. Moreover, there are scenarios
422 * where these checks can emit false-positive warnings in these
423 * drivers; so we avoid that by skipping them altogether.
424 */
425 WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
426 && current == policy->transition_task);
427
428 wait:
429 wait_event(policy->transition_wait, !policy->transition_ongoing);
430
431 spin_lock(&policy->transition_lock);
432
433 if (unlikely(policy->transition_ongoing)) {
434 spin_unlock(&policy->transition_lock);
435 goto wait;
436 }
437
438 policy->transition_ongoing = true;
439 policy->transition_task = current;
440
441 spin_unlock(&policy->transition_lock);
442
443 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
444 }
445 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
446
cpufreq_freq_transition_end(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs,int transition_failed)447 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
448 struct cpufreq_freqs *freqs, int transition_failed)
449 {
450 if (WARN_ON(!policy->transition_ongoing))
451 return;
452
453 cpufreq_notify_post_transition(policy, freqs, transition_failed);
454
455 arch_set_freq_scale(policy->related_cpus,
456 policy->cur,
457 policy->cpuinfo.max_freq);
458
459 spin_lock(&policy->transition_lock);
460 policy->transition_ongoing = false;
461 policy->transition_task = NULL;
462 spin_unlock(&policy->transition_lock);
463
464 wake_up(&policy->transition_wait);
465 }
466 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
467
468 /*
469 * Fast frequency switching status count. Positive means "enabled", negative
470 * means "disabled" and 0 means "not decided yet".
471 */
472 static int cpufreq_fast_switch_count;
473 static DEFINE_MUTEX(cpufreq_fast_switch_lock);
474
cpufreq_list_transition_notifiers(void)475 static void cpufreq_list_transition_notifiers(void)
476 {
477 struct notifier_block *nb;
478
479 pr_info("Registered transition notifiers:\n");
480
481 mutex_lock(&cpufreq_transition_notifier_list.mutex);
482
483 for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
484 pr_info("%pS\n", nb->notifier_call);
485
486 mutex_unlock(&cpufreq_transition_notifier_list.mutex);
487 }
488
489 /**
490 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
491 * @policy: cpufreq policy to enable fast frequency switching for.
492 *
493 * Try to enable fast frequency switching for @policy.
494 *
495 * The attempt will fail if there is at least one transition notifier registered
496 * at this point, as fast frequency switching is quite fundamentally at odds
497 * with transition notifiers. Thus if successful, it will make registration of
498 * transition notifiers fail going forward.
499 */
cpufreq_enable_fast_switch(struct cpufreq_policy * policy)500 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
501 {
502 lockdep_assert_held(&policy->rwsem);
503
504 if (!policy->fast_switch_possible)
505 return;
506
507 mutex_lock(&cpufreq_fast_switch_lock);
508 if (cpufreq_fast_switch_count >= 0) {
509 cpufreq_fast_switch_count++;
510 policy->fast_switch_enabled = true;
511 } else {
512 pr_warn("CPU%u: Fast frequency switching not enabled\n",
513 policy->cpu);
514 cpufreq_list_transition_notifiers();
515 }
516 mutex_unlock(&cpufreq_fast_switch_lock);
517 }
518 EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
519
520 /**
521 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
522 * @policy: cpufreq policy to disable fast frequency switching for.
523 */
cpufreq_disable_fast_switch(struct cpufreq_policy * policy)524 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
525 {
526 mutex_lock(&cpufreq_fast_switch_lock);
527 if (policy->fast_switch_enabled) {
528 policy->fast_switch_enabled = false;
529 if (!WARN_ON(cpufreq_fast_switch_count <= 0))
530 cpufreq_fast_switch_count--;
531 }
532 mutex_unlock(&cpufreq_fast_switch_lock);
533 }
534 EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
535
__resolve_freq(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)536 static unsigned int __resolve_freq(struct cpufreq_policy *policy,
537 unsigned int target_freq, unsigned int relation)
538 {
539 unsigned int idx;
540
541 target_freq = clamp_val(target_freq, policy->min, policy->max);
542
543 if (!policy->freq_table)
544 return target_freq;
545
546 idx = cpufreq_frequency_table_target(policy, target_freq, relation);
547 policy->cached_resolved_idx = idx;
548 policy->cached_target_freq = target_freq;
549 return policy->freq_table[idx].frequency;
550 }
551
552 /**
553 * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
554 * one.
555 * @policy: associated policy to interrogate
556 * @target_freq: target frequency to resolve.
557 *
558 * The target to driver frequency mapping is cached in the policy.
559 *
560 * Return: Lowest driver-supported frequency greater than or equal to the
561 * given target_freq, subject to policy (min/max) and driver limitations.
562 */
cpufreq_driver_resolve_freq(struct cpufreq_policy * policy,unsigned int target_freq)563 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
564 unsigned int target_freq)
565 {
566 return __resolve_freq(policy, target_freq, CPUFREQ_RELATION_LE);
567 }
568 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
569
cpufreq_policy_transition_delay_us(struct cpufreq_policy * policy)570 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
571 {
572 unsigned int latency;
573
574 if (policy->transition_delay_us)
575 return policy->transition_delay_us;
576
577 latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
578 if (latency) {
579 /*
580 * For platforms that can change the frequency very fast (< 10
581 * us), the above formula gives a decent transition delay. But
582 * for platforms where transition_latency is in milliseconds, it
583 * ends up giving unrealistic values.
584 *
585 * Cap the default transition delay to 10 ms, which seems to be
586 * a reasonable amount of time after which we should reevaluate
587 * the frequency.
588 */
589 return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000);
590 }
591
592 return LATENCY_MULTIPLIER;
593 }
594 EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
595
596 /*********************************************************************
597 * SYSFS INTERFACE *
598 *********************************************************************/
show_boost(struct kobject * kobj,struct kobj_attribute * attr,char * buf)599 static ssize_t show_boost(struct kobject *kobj,
600 struct kobj_attribute *attr, char *buf)
601 {
602 return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
603 }
604
store_boost(struct kobject * kobj,struct kobj_attribute * attr,const char * buf,size_t count)605 static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr,
606 const char *buf, size_t count)
607 {
608 int ret, enable;
609
610 ret = sscanf(buf, "%d", &enable);
611 if (ret != 1 || enable < 0 || enable > 1)
612 return -EINVAL;
613
614 if (cpufreq_boost_trigger_state(enable)) {
615 pr_err("%s: Cannot %s BOOST!\n",
616 __func__, enable ? "enable" : "disable");
617 return -EINVAL;
618 }
619
620 pr_debug("%s: cpufreq BOOST %s\n",
621 __func__, enable ? "enabled" : "disabled");
622
623 return count;
624 }
625 define_one_global_rw(boost);
626
show_local_boost(struct cpufreq_policy * policy,char * buf)627 static ssize_t show_local_boost(struct cpufreq_policy *policy, char *buf)
628 {
629 return sysfs_emit(buf, "%d\n", policy->boost_enabled);
630 }
631
store_local_boost(struct cpufreq_policy * policy,const char * buf,size_t count)632 static ssize_t store_local_boost(struct cpufreq_policy *policy,
633 const char *buf, size_t count)
634 {
635 int ret, enable;
636
637 ret = kstrtoint(buf, 10, &enable);
638 if (ret || enable < 0 || enable > 1)
639 return -EINVAL;
640
641 if (!cpufreq_driver->boost_enabled)
642 return -EINVAL;
643
644 if (policy->boost_enabled == enable)
645 return count;
646
647 policy->boost_enabled = enable;
648
649 cpus_read_lock();
650 ret = cpufreq_driver->set_boost(policy, enable);
651 cpus_read_unlock();
652
653 if (ret) {
654 policy->boost_enabled = !policy->boost_enabled;
655 return ret;
656 }
657
658 return count;
659 }
660
661 static struct freq_attr local_boost = __ATTR(boost, 0644, show_local_boost, store_local_boost);
662
find_governor(const char * str_governor)663 static struct cpufreq_governor *find_governor(const char *str_governor)
664 {
665 struct cpufreq_governor *t;
666
667 for_each_governor(t)
668 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
669 return t;
670
671 return NULL;
672 }
673
get_governor(const char * str_governor)674 static struct cpufreq_governor *get_governor(const char *str_governor)
675 {
676 struct cpufreq_governor *t;
677
678 mutex_lock(&cpufreq_governor_mutex);
679 t = find_governor(str_governor);
680 if (!t)
681 goto unlock;
682
683 if (!try_module_get(t->owner))
684 t = NULL;
685
686 unlock:
687 mutex_unlock(&cpufreq_governor_mutex);
688
689 return t;
690 }
691
cpufreq_parse_policy(char * str_governor)692 static unsigned int cpufreq_parse_policy(char *str_governor)
693 {
694 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN))
695 return CPUFREQ_POLICY_PERFORMANCE;
696
697 if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN))
698 return CPUFREQ_POLICY_POWERSAVE;
699
700 return CPUFREQ_POLICY_UNKNOWN;
701 }
702
703 /**
704 * cpufreq_parse_governor - parse a governor string only for has_target()
705 * @str_governor: Governor name.
706 */
cpufreq_parse_governor(char * str_governor)707 static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor)
708 {
709 struct cpufreq_governor *t;
710
711 t = get_governor(str_governor);
712 if (t)
713 return t;
714
715 if (request_module("cpufreq_%s", str_governor))
716 return NULL;
717
718 return get_governor(str_governor);
719 }
720
721 /*
722 * cpufreq_per_cpu_attr_read() / show_##file_name() -
723 * print out cpufreq information
724 *
725 * Write out information from cpufreq_driver->policy[cpu]; object must be
726 * "unsigned int".
727 */
728
729 #define show_one(file_name, object) \
730 static ssize_t show_##file_name \
731 (struct cpufreq_policy *policy, char *buf) \
732 { \
733 return sprintf(buf, "%u\n", policy->object); \
734 }
735
736 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
737 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
738 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
739 show_one(scaling_min_freq, min);
740 show_one(scaling_max_freq, max);
741
arch_freq_get_on_cpu(int cpu)742 __weak unsigned int arch_freq_get_on_cpu(int cpu)
743 {
744 return 0;
745 }
746
show_scaling_cur_freq(struct cpufreq_policy * policy,char * buf)747 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
748 {
749 ssize_t ret;
750 unsigned int freq;
751
752 freq = arch_freq_get_on_cpu(policy->cpu);
753 if (freq)
754 ret = sprintf(buf, "%u\n", freq);
755 else if (cpufreq_driver->setpolicy && cpufreq_driver->get)
756 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
757 else
758 ret = sprintf(buf, "%u\n", policy->cur);
759 return ret;
760 }
761
762 /*
763 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
764 */
765 #define store_one(file_name, object) \
766 static ssize_t store_##file_name \
767 (struct cpufreq_policy *policy, const char *buf, size_t count) \
768 { \
769 unsigned long val; \
770 int ret; \
771 \
772 ret = kstrtoul(buf, 0, &val); \
773 if (ret) \
774 return ret; \
775 \
776 ret = freq_qos_update_request(policy->object##_freq_req, val);\
777 return ret >= 0 ? count : ret; \
778 }
779
780 store_one(scaling_min_freq, min);
781 store_one(scaling_max_freq, max);
782
783 /*
784 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
785 */
show_cpuinfo_cur_freq(struct cpufreq_policy * policy,char * buf)786 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
787 char *buf)
788 {
789 unsigned int cur_freq = __cpufreq_get(policy);
790
791 if (cur_freq)
792 return sprintf(buf, "%u\n", cur_freq);
793
794 return sprintf(buf, "<unknown>\n");
795 }
796
797 /*
798 * show_scaling_governor - show the current policy for the specified CPU
799 */
show_scaling_governor(struct cpufreq_policy * policy,char * buf)800 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
801 {
802 if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
803 return sprintf(buf, "powersave\n");
804 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
805 return sprintf(buf, "performance\n");
806 else if (policy->governor)
807 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
808 policy->governor->name);
809 return -EINVAL;
810 }
811
812 /*
813 * store_scaling_governor - store policy for the specified CPU
814 */
store_scaling_governor(struct cpufreq_policy * policy,const char * buf,size_t count)815 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
816 const char *buf, size_t count)
817 {
818 char str_governor[16];
819 int ret;
820
821 ret = sscanf(buf, "%15s", str_governor);
822 if (ret != 1)
823 return -EINVAL;
824
825 if (cpufreq_driver->setpolicy) {
826 unsigned int new_pol;
827
828 new_pol = cpufreq_parse_policy(str_governor);
829 if (!new_pol)
830 return -EINVAL;
831
832 ret = cpufreq_set_policy(policy, NULL, new_pol);
833 } else {
834 struct cpufreq_governor *new_gov;
835
836 new_gov = cpufreq_parse_governor(str_governor);
837 if (!new_gov)
838 return -EINVAL;
839
840 ret = cpufreq_set_policy(policy, new_gov,
841 CPUFREQ_POLICY_UNKNOWN);
842
843 module_put(new_gov->owner);
844 }
845
846 return ret ? ret : count;
847 }
848
849 /*
850 * show_scaling_driver - show the cpufreq driver currently loaded
851 */
show_scaling_driver(struct cpufreq_policy * policy,char * buf)852 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
853 {
854 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
855 }
856
857 /*
858 * show_scaling_available_governors - show the available CPUfreq governors
859 */
show_scaling_available_governors(struct cpufreq_policy * policy,char * buf)860 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
861 char *buf)
862 {
863 ssize_t i = 0;
864 struct cpufreq_governor *t;
865
866 if (!has_target()) {
867 i += sprintf(buf, "performance powersave");
868 goto out;
869 }
870
871 mutex_lock(&cpufreq_governor_mutex);
872 for_each_governor(t) {
873 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
874 - (CPUFREQ_NAME_LEN + 2)))
875 break;
876 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
877 }
878 mutex_unlock(&cpufreq_governor_mutex);
879 out:
880 i += sprintf(&buf[i], "\n");
881 return i;
882 }
883
cpufreq_show_cpus(const struct cpumask * mask,char * buf)884 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
885 {
886 ssize_t i = 0;
887 unsigned int cpu;
888
889 for_each_cpu(cpu, mask) {
890 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u ", cpu);
891 if (i >= (PAGE_SIZE - 5))
892 break;
893 }
894
895 /* Remove the extra space at the end */
896 i--;
897
898 i += sprintf(&buf[i], "\n");
899 return i;
900 }
901 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
902
903 /*
904 * show_related_cpus - show the CPUs affected by each transition even if
905 * hw coordination is in use
906 */
show_related_cpus(struct cpufreq_policy * policy,char * buf)907 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
908 {
909 return cpufreq_show_cpus(policy->related_cpus, buf);
910 }
911
912 /*
913 * show_affected_cpus - show the CPUs affected by each transition
914 */
show_affected_cpus(struct cpufreq_policy * policy,char * buf)915 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
916 {
917 return cpufreq_show_cpus(policy->cpus, buf);
918 }
919
store_scaling_setspeed(struct cpufreq_policy * policy,const char * buf,size_t count)920 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
921 const char *buf, size_t count)
922 {
923 unsigned int freq = 0;
924 unsigned int ret;
925
926 if (!policy->governor || !policy->governor->store_setspeed)
927 return -EINVAL;
928
929 ret = sscanf(buf, "%u", &freq);
930 if (ret != 1)
931 return -EINVAL;
932
933 policy->governor->store_setspeed(policy, freq);
934
935 return count;
936 }
937
show_scaling_setspeed(struct cpufreq_policy * policy,char * buf)938 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
939 {
940 if (!policy->governor || !policy->governor->show_setspeed)
941 return sprintf(buf, "<unsupported>\n");
942
943 return policy->governor->show_setspeed(policy, buf);
944 }
945
946 /*
947 * show_bios_limit - show the current cpufreq HW/BIOS limitation
948 */
show_bios_limit(struct cpufreq_policy * policy,char * buf)949 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
950 {
951 unsigned int limit;
952 int ret;
953 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
954 if (!ret)
955 return sprintf(buf, "%u\n", limit);
956 return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
957 }
958
959 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
960 cpufreq_freq_attr_ro(cpuinfo_min_freq);
961 cpufreq_freq_attr_ro(cpuinfo_max_freq);
962 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
963 cpufreq_freq_attr_ro(scaling_available_governors);
964 cpufreq_freq_attr_ro(scaling_driver);
965 cpufreq_freq_attr_ro(scaling_cur_freq);
966 cpufreq_freq_attr_ro(bios_limit);
967 cpufreq_freq_attr_ro(related_cpus);
968 cpufreq_freq_attr_ro(affected_cpus);
969 cpufreq_freq_attr_rw(scaling_min_freq);
970 cpufreq_freq_attr_rw(scaling_max_freq);
971 cpufreq_freq_attr_rw(scaling_governor);
972 cpufreq_freq_attr_rw(scaling_setspeed);
973
974 static struct attribute *cpufreq_attrs[] = {
975 &cpuinfo_min_freq.attr,
976 &cpuinfo_max_freq.attr,
977 &cpuinfo_transition_latency.attr,
978 &scaling_min_freq.attr,
979 &scaling_max_freq.attr,
980 &affected_cpus.attr,
981 &related_cpus.attr,
982 &scaling_governor.attr,
983 &scaling_driver.attr,
984 &scaling_available_governors.attr,
985 &scaling_setspeed.attr,
986 NULL
987 };
988 ATTRIBUTE_GROUPS(cpufreq);
989
990 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
991 #define to_attr(a) container_of(a, struct freq_attr, attr)
992
show(struct kobject * kobj,struct attribute * attr,char * buf)993 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
994 {
995 struct cpufreq_policy *policy = to_policy(kobj);
996 struct freq_attr *fattr = to_attr(attr);
997 ssize_t ret = -EBUSY;
998
999 if (!fattr->show)
1000 return -EIO;
1001
1002 down_read(&policy->rwsem);
1003 if (likely(!policy_is_inactive(policy)))
1004 ret = fattr->show(policy, buf);
1005 up_read(&policy->rwsem);
1006
1007 return ret;
1008 }
1009
store(struct kobject * kobj,struct attribute * attr,const char * buf,size_t count)1010 static ssize_t store(struct kobject *kobj, struct attribute *attr,
1011 const char *buf, size_t count)
1012 {
1013 struct cpufreq_policy *policy = to_policy(kobj);
1014 struct freq_attr *fattr = to_attr(attr);
1015 ssize_t ret = -EBUSY;
1016
1017 if (!fattr->store)
1018 return -EIO;
1019
1020 down_write(&policy->rwsem);
1021 if (likely(!policy_is_inactive(policy)))
1022 ret = fattr->store(policy, buf, count);
1023 up_write(&policy->rwsem);
1024
1025 return ret;
1026 }
1027
cpufreq_sysfs_release(struct kobject * kobj)1028 static void cpufreq_sysfs_release(struct kobject *kobj)
1029 {
1030 struct cpufreq_policy *policy = to_policy(kobj);
1031 pr_debug("last reference is dropped\n");
1032 complete(&policy->kobj_unregister);
1033 }
1034
1035 static const struct sysfs_ops sysfs_ops = {
1036 .show = show,
1037 .store = store,
1038 };
1039
1040 static const struct kobj_type ktype_cpufreq = {
1041 .sysfs_ops = &sysfs_ops,
1042 .default_groups = cpufreq_groups,
1043 .release = cpufreq_sysfs_release,
1044 };
1045
add_cpu_dev_symlink(struct cpufreq_policy * policy,unsigned int cpu,struct device * dev)1046 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu,
1047 struct device *dev)
1048 {
1049 if (unlikely(!dev))
1050 return;
1051
1052 if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
1053 return;
1054
1055 dev_dbg(dev, "%s: Adding symlink\n", __func__);
1056 if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
1057 dev_err(dev, "cpufreq symlink creation failed\n");
1058 }
1059
remove_cpu_dev_symlink(struct cpufreq_policy * policy,int cpu,struct device * dev)1060 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu,
1061 struct device *dev)
1062 {
1063 dev_dbg(dev, "%s: Removing symlink\n", __func__);
1064 sysfs_remove_link(&dev->kobj, "cpufreq");
1065 cpumask_clear_cpu(cpu, policy->real_cpus);
1066 }
1067
cpufreq_add_dev_interface(struct cpufreq_policy * policy)1068 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
1069 {
1070 struct freq_attr **drv_attr;
1071 int ret = 0;
1072
1073 /* set up files for this cpu device */
1074 drv_attr = cpufreq_driver->attr;
1075 while (drv_attr && *drv_attr) {
1076 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
1077 if (ret)
1078 return ret;
1079 drv_attr++;
1080 }
1081 if (cpufreq_driver->get) {
1082 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
1083 if (ret)
1084 return ret;
1085 }
1086
1087 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
1088 if (ret)
1089 return ret;
1090
1091 if (cpufreq_driver->bios_limit) {
1092 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1093 if (ret)
1094 return ret;
1095 }
1096
1097 if (cpufreq_boost_supported()) {
1098 ret = sysfs_create_file(&policy->kobj, &local_boost.attr);
1099 if (ret)
1100 return ret;
1101 }
1102
1103 return 0;
1104 }
1105
cpufreq_init_policy(struct cpufreq_policy * policy)1106 static int cpufreq_init_policy(struct cpufreq_policy *policy)
1107 {
1108 struct cpufreq_governor *gov = NULL;
1109 unsigned int pol = CPUFREQ_POLICY_UNKNOWN;
1110 int ret;
1111
1112 if (has_target()) {
1113 /* Update policy governor to the one used before hotplug. */
1114 gov = get_governor(policy->last_governor);
1115 if (gov) {
1116 pr_debug("Restoring governor %s for cpu %d\n",
1117 gov->name, policy->cpu);
1118 } else {
1119 gov = get_governor(default_governor);
1120 }
1121
1122 if (!gov) {
1123 gov = cpufreq_default_governor();
1124 __module_get(gov->owner);
1125 }
1126
1127 } else {
1128
1129 /* Use the default policy if there is no last_policy. */
1130 if (policy->last_policy) {
1131 pol = policy->last_policy;
1132 } else {
1133 pol = cpufreq_parse_policy(default_governor);
1134 /*
1135 * In case the default governor is neither "performance"
1136 * nor "powersave", fall back to the initial policy
1137 * value set by the driver.
1138 */
1139 if (pol == CPUFREQ_POLICY_UNKNOWN)
1140 pol = policy->policy;
1141 }
1142 if (pol != CPUFREQ_POLICY_PERFORMANCE &&
1143 pol != CPUFREQ_POLICY_POWERSAVE)
1144 return -ENODATA;
1145 }
1146
1147 ret = cpufreq_set_policy(policy, gov, pol);
1148 if (gov)
1149 module_put(gov->owner);
1150
1151 return ret;
1152 }
1153
cpufreq_add_policy_cpu(struct cpufreq_policy * policy,unsigned int cpu)1154 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1155 {
1156 int ret = 0;
1157
1158 /* Has this CPU been taken care of already? */
1159 if (cpumask_test_cpu(cpu, policy->cpus))
1160 return 0;
1161
1162 down_write(&policy->rwsem);
1163 if (has_target())
1164 cpufreq_stop_governor(policy);
1165
1166 cpumask_set_cpu(cpu, policy->cpus);
1167
1168 if (has_target()) {
1169 ret = cpufreq_start_governor(policy);
1170 if (ret)
1171 pr_err("%s: Failed to start governor\n", __func__);
1172 }
1173 up_write(&policy->rwsem);
1174 return ret;
1175 }
1176
refresh_frequency_limits(struct cpufreq_policy * policy)1177 void refresh_frequency_limits(struct cpufreq_policy *policy)
1178 {
1179 if (!policy_is_inactive(policy)) {
1180 pr_debug("updating policy for CPU %u\n", policy->cpu);
1181
1182 cpufreq_set_policy(policy, policy->governor, policy->policy);
1183 }
1184 }
1185 EXPORT_SYMBOL(refresh_frequency_limits);
1186
handle_update(struct work_struct * work)1187 static void handle_update(struct work_struct *work)
1188 {
1189 struct cpufreq_policy *policy =
1190 container_of(work, struct cpufreq_policy, update);
1191
1192 pr_debug("handle_update for cpu %u called\n", policy->cpu);
1193 down_write(&policy->rwsem);
1194 refresh_frequency_limits(policy);
1195 up_write(&policy->rwsem);
1196 }
1197
cpufreq_notifier_min(struct notifier_block * nb,unsigned long freq,void * data)1198 static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq,
1199 void *data)
1200 {
1201 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);
1202
1203 schedule_work(&policy->update);
1204 return 0;
1205 }
1206
cpufreq_notifier_max(struct notifier_block * nb,unsigned long freq,void * data)1207 static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq,
1208 void *data)
1209 {
1210 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);
1211
1212 schedule_work(&policy->update);
1213 return 0;
1214 }
1215
cpufreq_policy_put_kobj(struct cpufreq_policy * policy)1216 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1217 {
1218 struct kobject *kobj;
1219 struct completion *cmp;
1220
1221 down_write(&policy->rwsem);
1222 cpufreq_stats_free_table(policy);
1223 kobj = &policy->kobj;
1224 cmp = &policy->kobj_unregister;
1225 up_write(&policy->rwsem);
1226 kobject_put(kobj);
1227
1228 /*
1229 * We need to make sure that the underlying kobj is
1230 * actually not referenced anymore by anybody before we
1231 * proceed with unloading.
1232 */
1233 pr_debug("waiting for dropping of refcount\n");
1234 wait_for_completion(cmp);
1235 pr_debug("wait complete\n");
1236 }
1237
cpufreq_policy_alloc(unsigned int cpu)1238 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1239 {
1240 struct cpufreq_policy *policy;
1241 struct device *dev = get_cpu_device(cpu);
1242 int ret;
1243
1244 if (!dev)
1245 return NULL;
1246
1247 policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1248 if (!policy)
1249 return NULL;
1250
1251 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1252 goto err_free_policy;
1253
1254 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1255 goto err_free_cpumask;
1256
1257 if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1258 goto err_free_rcpumask;
1259
1260 init_completion(&policy->kobj_unregister);
1261 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1262 cpufreq_global_kobject, "policy%u", cpu);
1263 if (ret) {
1264 dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret);
1265 /*
1266 * The entire policy object will be freed below, but the extra
1267 * memory allocated for the kobject name needs to be freed by
1268 * releasing the kobject.
1269 */
1270 kobject_put(&policy->kobj);
1271 goto err_free_real_cpus;
1272 }
1273
1274 freq_constraints_init(&policy->constraints);
1275
1276 policy->nb_min.notifier_call = cpufreq_notifier_min;
1277 policy->nb_max.notifier_call = cpufreq_notifier_max;
1278
1279 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN,
1280 &policy->nb_min);
1281 if (ret) {
1282 dev_err(dev, "Failed to register MIN QoS notifier: %d (CPU%u)\n",
1283 ret, cpu);
1284 goto err_kobj_remove;
1285 }
1286
1287 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX,
1288 &policy->nb_max);
1289 if (ret) {
1290 dev_err(dev, "Failed to register MAX QoS notifier: %d (CPU%u)\n",
1291 ret, cpu);
1292 goto err_min_qos_notifier;
1293 }
1294
1295 INIT_LIST_HEAD(&policy->policy_list);
1296 init_rwsem(&policy->rwsem);
1297 spin_lock_init(&policy->transition_lock);
1298 init_waitqueue_head(&policy->transition_wait);
1299 INIT_WORK(&policy->update, handle_update);
1300
1301 policy->cpu = cpu;
1302 return policy;
1303
1304 err_min_qos_notifier:
1305 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1306 &policy->nb_min);
1307 err_kobj_remove:
1308 cpufreq_policy_put_kobj(policy);
1309 err_free_real_cpus:
1310 free_cpumask_var(policy->real_cpus);
1311 err_free_rcpumask:
1312 free_cpumask_var(policy->related_cpus);
1313 err_free_cpumask:
1314 free_cpumask_var(policy->cpus);
1315 err_free_policy:
1316 kfree(policy);
1317
1318 return NULL;
1319 }
1320
cpufreq_policy_free(struct cpufreq_policy * policy)1321 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1322 {
1323 unsigned long flags;
1324 int cpu;
1325
1326 /*
1327 * The callers must ensure the policy is inactive by now, to avoid any
1328 * races with show()/store() callbacks.
1329 */
1330 if (unlikely(!policy_is_inactive(policy)))
1331 pr_warn("%s: Freeing active policy\n", __func__);
1332
1333 /* Remove policy from list */
1334 write_lock_irqsave(&cpufreq_driver_lock, flags);
1335 list_del(&policy->policy_list);
1336
1337 for_each_cpu(cpu, policy->related_cpus)
1338 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1339 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1340
1341 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX,
1342 &policy->nb_max);
1343 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1344 &policy->nb_min);
1345
1346 /* Cancel any pending policy->update work before freeing the policy. */
1347 cancel_work_sync(&policy->update);
1348
1349 if (policy->max_freq_req) {
1350 /*
1351 * Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY
1352 * notification, since CPUFREQ_CREATE_POLICY notification was
1353 * sent after adding max_freq_req earlier.
1354 */
1355 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1356 CPUFREQ_REMOVE_POLICY, policy);
1357 freq_qos_remove_request(policy->max_freq_req);
1358 }
1359
1360 freq_qos_remove_request(policy->min_freq_req);
1361 kfree(policy->min_freq_req);
1362
1363 cpufreq_policy_put_kobj(policy);
1364 free_cpumask_var(policy->real_cpus);
1365 free_cpumask_var(policy->related_cpus);
1366 free_cpumask_var(policy->cpus);
1367 kfree(policy);
1368 }
1369
cpufreq_online(unsigned int cpu)1370 static int cpufreq_online(unsigned int cpu)
1371 {
1372 struct cpufreq_policy *policy;
1373 bool new_policy;
1374 unsigned long flags;
1375 unsigned int j;
1376 int ret;
1377
1378 pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1379
1380 /* Check if this CPU already has a policy to manage it */
1381 policy = per_cpu(cpufreq_cpu_data, cpu);
1382 if (policy) {
1383 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1384 if (!policy_is_inactive(policy))
1385 return cpufreq_add_policy_cpu(policy, cpu);
1386
1387 /* This is the only online CPU for the policy. Start over. */
1388 new_policy = false;
1389 down_write(&policy->rwsem);
1390 policy->cpu = cpu;
1391 policy->governor = NULL;
1392 } else {
1393 new_policy = true;
1394 policy = cpufreq_policy_alloc(cpu);
1395 if (!policy)
1396 return -ENOMEM;
1397 down_write(&policy->rwsem);
1398 }
1399
1400 if (!new_policy && cpufreq_driver->online) {
1401 /* Recover policy->cpus using related_cpus */
1402 cpumask_copy(policy->cpus, policy->related_cpus);
1403
1404 ret = cpufreq_driver->online(policy);
1405 if (ret) {
1406 pr_debug("%s: %d: initialization failed\n", __func__,
1407 __LINE__);
1408 goto out_exit_policy;
1409 }
1410 } else {
1411 cpumask_copy(policy->cpus, cpumask_of(cpu));
1412
1413 /*
1414 * Call driver. From then on the cpufreq must be able
1415 * to accept all calls to ->verify and ->setpolicy for this CPU.
1416 */
1417 ret = cpufreq_driver->init(policy);
1418 if (ret) {
1419 pr_debug("%s: %d: initialization failed\n", __func__,
1420 __LINE__);
1421 goto out_free_policy;
1422 }
1423
1424 /* Let the per-policy boost flag mirror the cpufreq_driver boost during init */
1425 policy->boost_enabled = cpufreq_boost_enabled() && policy_has_boost_freq(policy);
1426
1427 /*
1428 * The initialization has succeeded and the policy is online.
1429 * If there is a problem with its frequency table, take it
1430 * offline and drop it.
1431 */
1432 ret = cpufreq_table_validate_and_sort(policy);
1433 if (ret)
1434 goto out_offline_policy;
1435
1436 /* related_cpus should at least include policy->cpus. */
1437 cpumask_copy(policy->related_cpus, policy->cpus);
1438 }
1439
1440 /*
1441 * affected cpus must always be the one, which are online. We aren't
1442 * managing offline cpus here.
1443 */
1444 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1445
1446 if (new_policy) {
1447 for_each_cpu(j, policy->related_cpus) {
1448 per_cpu(cpufreq_cpu_data, j) = policy;
1449 add_cpu_dev_symlink(policy, j, get_cpu_device(j));
1450 }
1451
1452 policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req),
1453 GFP_KERNEL);
1454 if (!policy->min_freq_req) {
1455 ret = -ENOMEM;
1456 goto out_destroy_policy;
1457 }
1458
1459 ret = freq_qos_add_request(&policy->constraints,
1460 policy->min_freq_req, FREQ_QOS_MIN,
1461 FREQ_QOS_MIN_DEFAULT_VALUE);
1462 if (ret < 0) {
1463 /*
1464 * So we don't call freq_qos_remove_request() for an
1465 * uninitialized request.
1466 */
1467 kfree(policy->min_freq_req);
1468 policy->min_freq_req = NULL;
1469 goto out_destroy_policy;
1470 }
1471
1472 /*
1473 * This must be initialized right here to avoid calling
1474 * freq_qos_remove_request() on uninitialized request in case
1475 * of errors.
1476 */
1477 policy->max_freq_req = policy->min_freq_req + 1;
1478
1479 ret = freq_qos_add_request(&policy->constraints,
1480 policy->max_freq_req, FREQ_QOS_MAX,
1481 FREQ_QOS_MAX_DEFAULT_VALUE);
1482 if (ret < 0) {
1483 policy->max_freq_req = NULL;
1484 goto out_destroy_policy;
1485 }
1486
1487 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1488 CPUFREQ_CREATE_POLICY, policy);
1489 }
1490
1491 if (cpufreq_driver->get && has_target()) {
1492 policy->cur = cpufreq_driver->get(policy->cpu);
1493 if (!policy->cur) {
1494 ret = -EIO;
1495 pr_err("%s: ->get() failed\n", __func__);
1496 goto out_destroy_policy;
1497 }
1498 }
1499
1500 /*
1501 * Sometimes boot loaders set CPU frequency to a value outside of
1502 * frequency table present with cpufreq core. In such cases CPU might be
1503 * unstable if it has to run on that frequency for long duration of time
1504 * and so its better to set it to a frequency which is specified in
1505 * freq-table. This also makes cpufreq stats inconsistent as
1506 * cpufreq-stats would fail to register because current frequency of CPU
1507 * isn't found in freq-table.
1508 *
1509 * Because we don't want this change to effect boot process badly, we go
1510 * for the next freq which is >= policy->cur ('cur' must be set by now,
1511 * otherwise we will end up setting freq to lowest of the table as 'cur'
1512 * is initialized to zero).
1513 *
1514 * We are passing target-freq as "policy->cur - 1" otherwise
1515 * __cpufreq_driver_target() would simply fail, as policy->cur will be
1516 * equal to target-freq.
1517 */
1518 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1519 && has_target()) {
1520 unsigned int old_freq = policy->cur;
1521
1522 /* Are we running at unknown frequency ? */
1523 ret = cpufreq_frequency_table_get_index(policy, old_freq);
1524 if (ret == -EINVAL) {
1525 ret = __cpufreq_driver_target(policy, old_freq - 1,
1526 CPUFREQ_RELATION_L);
1527
1528 /*
1529 * Reaching here after boot in a few seconds may not
1530 * mean that system will remain stable at "unknown"
1531 * frequency for longer duration. Hence, a BUG_ON().
1532 */
1533 BUG_ON(ret);
1534 pr_info("%s: CPU%d: Running at unlisted initial frequency: %u KHz, changing to: %u KHz\n",
1535 __func__, policy->cpu, old_freq, policy->cur);
1536 }
1537 }
1538
1539 if (new_policy) {
1540 ret = cpufreq_add_dev_interface(policy);
1541 if (ret)
1542 goto out_destroy_policy;
1543
1544 cpufreq_stats_create_table(policy);
1545
1546 write_lock_irqsave(&cpufreq_driver_lock, flags);
1547 list_add(&policy->policy_list, &cpufreq_policy_list);
1548 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1549
1550 /*
1551 * Register with the energy model before
1552 * sched_cpufreq_governor_change() is called, which will result
1553 * in rebuilding of the sched domains, which should only be done
1554 * once the energy model is properly initialized for the policy
1555 * first.
1556 *
1557 * Also, this should be called before the policy is registered
1558 * with cooling framework.
1559 */
1560 if (cpufreq_driver->register_em)
1561 cpufreq_driver->register_em(policy);
1562 }
1563
1564 ret = cpufreq_init_policy(policy);
1565 if (ret) {
1566 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1567 __func__, cpu, ret);
1568 goto out_destroy_policy;
1569 }
1570
1571 up_write(&policy->rwsem);
1572
1573 kobject_uevent(&policy->kobj, KOBJ_ADD);
1574
1575 /* Callback for handling stuff after policy is ready */
1576 if (cpufreq_driver->ready)
1577 cpufreq_driver->ready(policy);
1578
1579 /* Register cpufreq cooling only for a new policy */
1580 if (new_policy && cpufreq_thermal_control_enabled(cpufreq_driver))
1581 policy->cdev = of_cpufreq_cooling_register(policy);
1582
1583 pr_debug("initialization complete\n");
1584
1585 return 0;
1586
1587 out_destroy_policy:
1588 for_each_cpu(j, policy->real_cpus)
1589 remove_cpu_dev_symlink(policy, j, get_cpu_device(j));
1590
1591 out_offline_policy:
1592 if (cpufreq_driver->offline)
1593 cpufreq_driver->offline(policy);
1594
1595 out_exit_policy:
1596 if (cpufreq_driver->exit)
1597 cpufreq_driver->exit(policy);
1598
1599 out_free_policy:
1600 cpumask_clear(policy->cpus);
1601 up_write(&policy->rwsem);
1602
1603 cpufreq_policy_free(policy);
1604 return ret;
1605 }
1606
1607 /**
1608 * cpufreq_add_dev - the cpufreq interface for a CPU device.
1609 * @dev: CPU device.
1610 * @sif: Subsystem interface structure pointer (not used)
1611 */
cpufreq_add_dev(struct device * dev,struct subsys_interface * sif)1612 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1613 {
1614 struct cpufreq_policy *policy;
1615 unsigned cpu = dev->id;
1616 int ret;
1617
1618 dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1619
1620 if (cpu_online(cpu)) {
1621 ret = cpufreq_online(cpu);
1622 if (ret)
1623 return ret;
1624 }
1625
1626 /* Create sysfs link on CPU registration */
1627 policy = per_cpu(cpufreq_cpu_data, cpu);
1628 if (policy)
1629 add_cpu_dev_symlink(policy, cpu, dev);
1630
1631 return 0;
1632 }
1633
__cpufreq_offline(unsigned int cpu,struct cpufreq_policy * policy)1634 static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy)
1635 {
1636 int ret;
1637
1638 if (has_target())
1639 cpufreq_stop_governor(policy);
1640
1641 cpumask_clear_cpu(cpu, policy->cpus);
1642
1643 if (!policy_is_inactive(policy)) {
1644 /* Nominate a new CPU if necessary. */
1645 if (cpu == policy->cpu)
1646 policy->cpu = cpumask_any(policy->cpus);
1647
1648 /* Start the governor again for the active policy. */
1649 if (has_target()) {
1650 ret = cpufreq_start_governor(policy);
1651 if (ret)
1652 pr_err("%s: Failed to start governor\n", __func__);
1653 }
1654
1655 return;
1656 }
1657
1658 if (has_target())
1659 strncpy(policy->last_governor, policy->governor->name,
1660 CPUFREQ_NAME_LEN);
1661 else
1662 policy->last_policy = policy->policy;
1663
1664 if (has_target())
1665 cpufreq_exit_governor(policy);
1666
1667 /*
1668 * Perform the ->offline() during light-weight tear-down, as
1669 * that allows fast recovery when the CPU comes back.
1670 */
1671 if (cpufreq_driver->offline) {
1672 cpufreq_driver->offline(policy);
1673 return;
1674 }
1675
1676 if (cpufreq_driver->exit)
1677 cpufreq_driver->exit(policy);
1678
1679 policy->freq_table = NULL;
1680 }
1681
cpufreq_offline(unsigned int cpu)1682 static int cpufreq_offline(unsigned int cpu)
1683 {
1684 struct cpufreq_policy *policy;
1685
1686 pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1687
1688 policy = cpufreq_cpu_get_raw(cpu);
1689 if (!policy) {
1690 pr_debug("%s: No cpu_data found\n", __func__);
1691 return 0;
1692 }
1693
1694 down_write(&policy->rwsem);
1695
1696 __cpufreq_offline(cpu, policy);
1697
1698 up_write(&policy->rwsem);
1699 return 0;
1700 }
1701
1702 /*
1703 * cpufreq_remove_dev - remove a CPU device
1704 *
1705 * Removes the cpufreq interface for a CPU device.
1706 */
cpufreq_remove_dev(struct device * dev,struct subsys_interface * sif)1707 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1708 {
1709 unsigned int cpu = dev->id;
1710 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1711
1712 if (!policy)
1713 return;
1714
1715 down_write(&policy->rwsem);
1716
1717 if (cpu_online(cpu))
1718 __cpufreq_offline(cpu, policy);
1719
1720 remove_cpu_dev_symlink(policy, cpu, dev);
1721
1722 if (!cpumask_empty(policy->real_cpus)) {
1723 up_write(&policy->rwsem);
1724 return;
1725 }
1726
1727 /*
1728 * Unregister cpufreq cooling once all the CPUs of the policy are
1729 * removed.
1730 */
1731 if (cpufreq_thermal_control_enabled(cpufreq_driver)) {
1732 cpufreq_cooling_unregister(policy->cdev);
1733 policy->cdev = NULL;
1734 }
1735
1736 /* We did light-weight exit earlier, do full tear down now */
1737 if (cpufreq_driver->offline && cpufreq_driver->exit)
1738 cpufreq_driver->exit(policy);
1739
1740 up_write(&policy->rwsem);
1741
1742 cpufreq_policy_free(policy);
1743 }
1744
1745 /**
1746 * cpufreq_out_of_sync - Fix up actual and saved CPU frequency difference.
1747 * @policy: Policy managing CPUs.
1748 * @new_freq: New CPU frequency.
1749 *
1750 * Adjust to the current frequency first and clean up later by either calling
1751 * cpufreq_update_policy(), or scheduling handle_update().
1752 */
cpufreq_out_of_sync(struct cpufreq_policy * policy,unsigned int new_freq)1753 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1754 unsigned int new_freq)
1755 {
1756 struct cpufreq_freqs freqs;
1757
1758 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1759 policy->cur, new_freq);
1760
1761 freqs.old = policy->cur;
1762 freqs.new = new_freq;
1763
1764 cpufreq_freq_transition_begin(policy, &freqs);
1765 cpufreq_freq_transition_end(policy, &freqs, 0);
1766 }
1767
cpufreq_verify_current_freq(struct cpufreq_policy * policy,bool update)1768 static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)
1769 {
1770 unsigned int new_freq;
1771
1772 new_freq = cpufreq_driver->get(policy->cpu);
1773 if (!new_freq)
1774 return 0;
1775
1776 /*
1777 * If fast frequency switching is used with the given policy, the check
1778 * against policy->cur is pointless, so skip it in that case.
1779 */
1780 if (policy->fast_switch_enabled || !has_target())
1781 return new_freq;
1782
1783 if (policy->cur != new_freq) {
1784 /*
1785 * For some platforms, the frequency returned by hardware may be
1786 * slightly different from what is provided in the frequency
1787 * table, for example hardware may return 499 MHz instead of 500
1788 * MHz. In such cases it is better to avoid getting into
1789 * unnecessary frequency updates.
1790 */
1791 if (abs(policy->cur - new_freq) < KHZ_PER_MHZ)
1792 return policy->cur;
1793
1794 cpufreq_out_of_sync(policy, new_freq);
1795 if (update)
1796 schedule_work(&policy->update);
1797 }
1798
1799 return new_freq;
1800 }
1801
1802 /**
1803 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1804 * @cpu: CPU number
1805 *
1806 * This is the last known freq, without actually getting it from the driver.
1807 * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1808 */
cpufreq_quick_get(unsigned int cpu)1809 unsigned int cpufreq_quick_get(unsigned int cpu)
1810 {
1811 struct cpufreq_policy *policy;
1812 unsigned int ret_freq = 0;
1813 unsigned long flags;
1814
1815 read_lock_irqsave(&cpufreq_driver_lock, flags);
1816
1817 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
1818 ret_freq = cpufreq_driver->get(cpu);
1819 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1820 return ret_freq;
1821 }
1822
1823 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1824
1825 policy = cpufreq_cpu_get(cpu);
1826 if (policy) {
1827 ret_freq = policy->cur;
1828 cpufreq_cpu_put(policy);
1829 }
1830
1831 return ret_freq;
1832 }
1833 EXPORT_SYMBOL(cpufreq_quick_get);
1834
1835 /**
1836 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1837 * @cpu: CPU number
1838 *
1839 * Just return the max possible frequency for a given CPU.
1840 */
cpufreq_quick_get_max(unsigned int cpu)1841 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1842 {
1843 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1844 unsigned int ret_freq = 0;
1845
1846 if (policy) {
1847 ret_freq = policy->max;
1848 cpufreq_cpu_put(policy);
1849 }
1850
1851 return ret_freq;
1852 }
1853 EXPORT_SYMBOL(cpufreq_quick_get_max);
1854
1855 /**
1856 * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU
1857 * @cpu: CPU number
1858 *
1859 * The default return value is the max_freq field of cpuinfo.
1860 */
cpufreq_get_hw_max_freq(unsigned int cpu)1861 __weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
1862 {
1863 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1864 unsigned int ret_freq = 0;
1865
1866 if (policy) {
1867 ret_freq = policy->cpuinfo.max_freq;
1868 cpufreq_cpu_put(policy);
1869 }
1870
1871 return ret_freq;
1872 }
1873 EXPORT_SYMBOL(cpufreq_get_hw_max_freq);
1874
__cpufreq_get(struct cpufreq_policy * policy)1875 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1876 {
1877 if (unlikely(policy_is_inactive(policy)))
1878 return 0;
1879
1880 return cpufreq_verify_current_freq(policy, true);
1881 }
1882
1883 /**
1884 * cpufreq_get - get the current CPU frequency (in kHz)
1885 * @cpu: CPU number
1886 *
1887 * Get the CPU current (static) CPU frequency
1888 */
cpufreq_get(unsigned int cpu)1889 unsigned int cpufreq_get(unsigned int cpu)
1890 {
1891 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1892 unsigned int ret_freq = 0;
1893
1894 if (policy) {
1895 down_read(&policy->rwsem);
1896 if (cpufreq_driver->get)
1897 ret_freq = __cpufreq_get(policy);
1898 up_read(&policy->rwsem);
1899
1900 cpufreq_cpu_put(policy);
1901 }
1902
1903 return ret_freq;
1904 }
1905 EXPORT_SYMBOL(cpufreq_get);
1906
1907 static struct subsys_interface cpufreq_interface = {
1908 .name = "cpufreq",
1909 .subsys = &cpu_subsys,
1910 .add_dev = cpufreq_add_dev,
1911 .remove_dev = cpufreq_remove_dev,
1912 };
1913
1914 /*
1915 * In case platform wants some specific frequency to be configured
1916 * during suspend..
1917 */
cpufreq_generic_suspend(struct cpufreq_policy * policy)1918 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1919 {
1920 int ret;
1921
1922 if (!policy->suspend_freq) {
1923 pr_debug("%s: suspend_freq not defined\n", __func__);
1924 return 0;
1925 }
1926
1927 pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1928 policy->suspend_freq);
1929
1930 ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1931 CPUFREQ_RELATION_H);
1932 if (ret)
1933 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1934 __func__, policy->suspend_freq, ret);
1935
1936 return ret;
1937 }
1938 EXPORT_SYMBOL(cpufreq_generic_suspend);
1939
1940 /**
1941 * cpufreq_suspend() - Suspend CPUFreq governors.
1942 *
1943 * Called during system wide Suspend/Hibernate cycles for suspending governors
1944 * as some platforms can't change frequency after this point in suspend cycle.
1945 * Because some of the devices (like: i2c, regulators, etc) they use for
1946 * changing frequency are suspended quickly after this point.
1947 */
cpufreq_suspend(void)1948 void cpufreq_suspend(void)
1949 {
1950 struct cpufreq_policy *policy;
1951
1952 if (!cpufreq_driver)
1953 return;
1954
1955 if (!has_target() && !cpufreq_driver->suspend)
1956 goto suspend;
1957
1958 pr_debug("%s: Suspending Governors\n", __func__);
1959
1960 for_each_active_policy(policy) {
1961 if (has_target()) {
1962 down_write(&policy->rwsem);
1963 cpufreq_stop_governor(policy);
1964 up_write(&policy->rwsem);
1965 }
1966
1967 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
1968 pr_err("%s: Failed to suspend driver: %s\n", __func__,
1969 cpufreq_driver->name);
1970 }
1971
1972 suspend:
1973 cpufreq_suspended = true;
1974 }
1975
1976 /**
1977 * cpufreq_resume() - Resume CPUFreq governors.
1978 *
1979 * Called during system wide Suspend/Hibernate cycle for resuming governors that
1980 * are suspended with cpufreq_suspend().
1981 */
cpufreq_resume(void)1982 void cpufreq_resume(void)
1983 {
1984 struct cpufreq_policy *policy;
1985 int ret;
1986
1987 if (!cpufreq_driver)
1988 return;
1989
1990 if (unlikely(!cpufreq_suspended))
1991 return;
1992
1993 cpufreq_suspended = false;
1994
1995 if (!has_target() && !cpufreq_driver->resume)
1996 return;
1997
1998 pr_debug("%s: Resuming Governors\n", __func__);
1999
2000 for_each_active_policy(policy) {
2001 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
2002 pr_err("%s: Failed to resume driver: %s\n", __func__,
2003 cpufreq_driver->name);
2004 } else if (has_target()) {
2005 down_write(&policy->rwsem);
2006 ret = cpufreq_start_governor(policy);
2007 up_write(&policy->rwsem);
2008
2009 if (ret)
2010 pr_err("%s: Failed to start governor for CPU%u's policy\n",
2011 __func__, policy->cpu);
2012 }
2013 }
2014 }
2015
2016 /**
2017 * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones.
2018 * @flags: Flags to test against the current cpufreq driver's flags.
2019 *
2020 * Assumes that the driver is there, so callers must ensure that this is the
2021 * case.
2022 */
cpufreq_driver_test_flags(u16 flags)2023 bool cpufreq_driver_test_flags(u16 flags)
2024 {
2025 return !!(cpufreq_driver->flags & flags);
2026 }
2027
2028 /**
2029 * cpufreq_get_current_driver - Return the current driver's name.
2030 *
2031 * Return the name string of the currently registered cpufreq driver or NULL if
2032 * none.
2033 */
cpufreq_get_current_driver(void)2034 const char *cpufreq_get_current_driver(void)
2035 {
2036 if (cpufreq_driver)
2037 return cpufreq_driver->name;
2038
2039 return NULL;
2040 }
2041 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
2042
2043 /**
2044 * cpufreq_get_driver_data - Return current driver data.
2045 *
2046 * Return the private data of the currently registered cpufreq driver, or NULL
2047 * if no cpufreq driver has been registered.
2048 */
cpufreq_get_driver_data(void)2049 void *cpufreq_get_driver_data(void)
2050 {
2051 if (cpufreq_driver)
2052 return cpufreq_driver->driver_data;
2053
2054 return NULL;
2055 }
2056 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
2057
2058 /*********************************************************************
2059 * NOTIFIER LISTS INTERFACE *
2060 *********************************************************************/
2061
2062 /**
2063 * cpufreq_register_notifier - Register a notifier with cpufreq.
2064 * @nb: notifier function to register.
2065 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2066 *
2067 * Add a notifier to one of two lists: either a list of notifiers that run on
2068 * clock rate changes (once before and once after every transition), or a list
2069 * of notifiers that ron on cpufreq policy changes.
2070 *
2071 * This function may sleep and it has the same return values as
2072 * blocking_notifier_chain_register().
2073 */
cpufreq_register_notifier(struct notifier_block * nb,unsigned int list)2074 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
2075 {
2076 int ret;
2077
2078 if (cpufreq_disabled())
2079 return -EINVAL;
2080
2081 switch (list) {
2082 case CPUFREQ_TRANSITION_NOTIFIER:
2083 mutex_lock(&cpufreq_fast_switch_lock);
2084
2085 if (cpufreq_fast_switch_count > 0) {
2086 mutex_unlock(&cpufreq_fast_switch_lock);
2087 return -EBUSY;
2088 }
2089 ret = srcu_notifier_chain_register(
2090 &cpufreq_transition_notifier_list, nb);
2091 if (!ret)
2092 cpufreq_fast_switch_count--;
2093
2094 mutex_unlock(&cpufreq_fast_switch_lock);
2095 break;
2096 case CPUFREQ_POLICY_NOTIFIER:
2097 ret = blocking_notifier_chain_register(
2098 &cpufreq_policy_notifier_list, nb);
2099 break;
2100 default:
2101 ret = -EINVAL;
2102 }
2103
2104 return ret;
2105 }
2106 EXPORT_SYMBOL(cpufreq_register_notifier);
2107
2108 /**
2109 * cpufreq_unregister_notifier - Unregister a notifier from cpufreq.
2110 * @nb: notifier block to be unregistered.
2111 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2112 *
2113 * Remove a notifier from one of the cpufreq notifier lists.
2114 *
2115 * This function may sleep and it has the same return values as
2116 * blocking_notifier_chain_unregister().
2117 */
cpufreq_unregister_notifier(struct notifier_block * nb,unsigned int list)2118 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
2119 {
2120 int ret;
2121
2122 if (cpufreq_disabled())
2123 return -EINVAL;
2124
2125 switch (list) {
2126 case CPUFREQ_TRANSITION_NOTIFIER:
2127 mutex_lock(&cpufreq_fast_switch_lock);
2128
2129 ret = srcu_notifier_chain_unregister(
2130 &cpufreq_transition_notifier_list, nb);
2131 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
2132 cpufreq_fast_switch_count++;
2133
2134 mutex_unlock(&cpufreq_fast_switch_lock);
2135 break;
2136 case CPUFREQ_POLICY_NOTIFIER:
2137 ret = blocking_notifier_chain_unregister(
2138 &cpufreq_policy_notifier_list, nb);
2139 break;
2140 default:
2141 ret = -EINVAL;
2142 }
2143
2144 return ret;
2145 }
2146 EXPORT_SYMBOL(cpufreq_unregister_notifier);
2147
2148
2149 /*********************************************************************
2150 * GOVERNORS *
2151 *********************************************************************/
2152
2153 /**
2154 * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
2155 * @policy: cpufreq policy to switch the frequency for.
2156 * @target_freq: New frequency to set (may be approximate).
2157 *
2158 * Carry out a fast frequency switch without sleeping.
2159 *
2160 * The driver's ->fast_switch() callback invoked by this function must be
2161 * suitable for being called from within RCU-sched read-side critical sections
2162 * and it is expected to select the minimum available frequency greater than or
2163 * equal to @target_freq (CPUFREQ_RELATION_L).
2164 *
2165 * This function must not be called if policy->fast_switch_enabled is unset.
2166 *
2167 * Governors calling this function must guarantee that it will never be invoked
2168 * twice in parallel for the same policy and that it will never be called in
2169 * parallel with either ->target() or ->target_index() for the same policy.
2170 *
2171 * Returns the actual frequency set for the CPU.
2172 *
2173 * If 0 is returned by the driver's ->fast_switch() callback to indicate an
2174 * error condition, the hardware configuration must be preserved.
2175 */
cpufreq_driver_fast_switch(struct cpufreq_policy * policy,unsigned int target_freq)2176 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
2177 unsigned int target_freq)
2178 {
2179 unsigned int freq;
2180 int cpu;
2181
2182 target_freq = clamp_val(target_freq, policy->min, policy->max);
2183 freq = cpufreq_driver->fast_switch(policy, target_freq);
2184
2185 if (!freq)
2186 return 0;
2187
2188 policy->cur = freq;
2189 arch_set_freq_scale(policy->related_cpus, freq,
2190 policy->cpuinfo.max_freq);
2191 cpufreq_stats_record_transition(policy, freq);
2192
2193 if (trace_cpu_frequency_enabled()) {
2194 for_each_cpu(cpu, policy->cpus)
2195 trace_cpu_frequency(freq, cpu);
2196 }
2197
2198 return freq;
2199 }
2200 EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
2201
2202 /**
2203 * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go.
2204 * @cpu: Target CPU.
2205 * @min_perf: Minimum (required) performance level (units of @capacity).
2206 * @target_perf: Target (desired) performance level (units of @capacity).
2207 * @capacity: Capacity of the target CPU.
2208 *
2209 * Carry out a fast performance level switch of @cpu without sleeping.
2210 *
2211 * The driver's ->adjust_perf() callback invoked by this function must be
2212 * suitable for being called from within RCU-sched read-side critical sections
2213 * and it is expected to select a suitable performance level equal to or above
2214 * @min_perf and preferably equal to or below @target_perf.
2215 *
2216 * This function must not be called if policy->fast_switch_enabled is unset.
2217 *
2218 * Governors calling this function must guarantee that it will never be invoked
2219 * twice in parallel for the same CPU and that it will never be called in
2220 * parallel with either ->target() or ->target_index() or ->fast_switch() for
2221 * the same CPU.
2222 */
cpufreq_driver_adjust_perf(unsigned int cpu,unsigned long min_perf,unsigned long target_perf,unsigned long capacity)2223 void cpufreq_driver_adjust_perf(unsigned int cpu,
2224 unsigned long min_perf,
2225 unsigned long target_perf,
2226 unsigned long capacity)
2227 {
2228 cpufreq_driver->adjust_perf(cpu, min_perf, target_perf, capacity);
2229 }
2230
2231 /**
2232 * cpufreq_driver_has_adjust_perf - Check "direct fast switch" callback.
2233 *
2234 * Return 'true' if the ->adjust_perf callback is present for the
2235 * current driver or 'false' otherwise.
2236 */
cpufreq_driver_has_adjust_perf(void)2237 bool cpufreq_driver_has_adjust_perf(void)
2238 {
2239 return !!cpufreq_driver->adjust_perf;
2240 }
2241
2242 /* Must set freqs->new to intermediate frequency */
__target_intermediate(struct cpufreq_policy * policy,struct cpufreq_freqs * freqs,int index)2243 static int __target_intermediate(struct cpufreq_policy *policy,
2244 struct cpufreq_freqs *freqs, int index)
2245 {
2246 int ret;
2247
2248 freqs->new = cpufreq_driver->get_intermediate(policy, index);
2249
2250 /* We don't need to switch to intermediate freq */
2251 if (!freqs->new)
2252 return 0;
2253
2254 pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
2255 __func__, policy->cpu, freqs->old, freqs->new);
2256
2257 cpufreq_freq_transition_begin(policy, freqs);
2258 ret = cpufreq_driver->target_intermediate(policy, index);
2259 cpufreq_freq_transition_end(policy, freqs, ret);
2260
2261 if (ret)
2262 pr_err("%s: Failed to change to intermediate frequency: %d\n",
2263 __func__, ret);
2264
2265 return ret;
2266 }
2267
__target_index(struct cpufreq_policy * policy,int index)2268 static int __target_index(struct cpufreq_policy *policy, int index)
2269 {
2270 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
2271 unsigned int restore_freq, intermediate_freq = 0;
2272 unsigned int newfreq = policy->freq_table[index].frequency;
2273 int retval = -EINVAL;
2274 bool notify;
2275
2276 if (newfreq == policy->cur)
2277 return 0;
2278
2279 /* Save last value to restore later on errors */
2280 restore_freq = policy->cur;
2281
2282 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
2283 if (notify) {
2284 /* Handle switching to intermediate frequency */
2285 if (cpufreq_driver->get_intermediate) {
2286 retval = __target_intermediate(policy, &freqs, index);
2287 if (retval)
2288 return retval;
2289
2290 intermediate_freq = freqs.new;
2291 /* Set old freq to intermediate */
2292 if (intermediate_freq)
2293 freqs.old = freqs.new;
2294 }
2295
2296 freqs.new = newfreq;
2297 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
2298 __func__, policy->cpu, freqs.old, freqs.new);
2299
2300 cpufreq_freq_transition_begin(policy, &freqs);
2301 }
2302
2303 retval = cpufreq_driver->target_index(policy, index);
2304 if (retval)
2305 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
2306 retval);
2307
2308 if (notify) {
2309 cpufreq_freq_transition_end(policy, &freqs, retval);
2310
2311 /*
2312 * Failed after setting to intermediate freq? Driver should have
2313 * reverted back to initial frequency and so should we. Check
2314 * here for intermediate_freq instead of get_intermediate, in
2315 * case we haven't switched to intermediate freq at all.
2316 */
2317 if (unlikely(retval && intermediate_freq)) {
2318 freqs.old = intermediate_freq;
2319 freqs.new = restore_freq;
2320 cpufreq_freq_transition_begin(policy, &freqs);
2321 cpufreq_freq_transition_end(policy, &freqs, 0);
2322 }
2323 }
2324
2325 return retval;
2326 }
2327
__cpufreq_driver_target(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)2328 int __cpufreq_driver_target(struct cpufreq_policy *policy,
2329 unsigned int target_freq,
2330 unsigned int relation)
2331 {
2332 unsigned int old_target_freq = target_freq;
2333
2334 if (cpufreq_disabled())
2335 return -ENODEV;
2336
2337 target_freq = __resolve_freq(policy, target_freq, relation);
2338
2339 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
2340 policy->cpu, target_freq, relation, old_target_freq);
2341
2342 /*
2343 * This might look like a redundant call as we are checking it again
2344 * after finding index. But it is left intentionally for cases where
2345 * exactly same freq is called again and so we can save on few function
2346 * calls.
2347 */
2348 if (target_freq == policy->cur &&
2349 !(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS))
2350 return 0;
2351
2352 if (cpufreq_driver->target) {
2353 /*
2354 * If the driver hasn't setup a single inefficient frequency,
2355 * it's unlikely it knows how to decode CPUFREQ_RELATION_E.
2356 */
2357 if (!policy->efficiencies_available)
2358 relation &= ~CPUFREQ_RELATION_E;
2359
2360 return cpufreq_driver->target(policy, target_freq, relation);
2361 }
2362
2363 if (!cpufreq_driver->target_index)
2364 return -EINVAL;
2365
2366 return __target_index(policy, policy->cached_resolved_idx);
2367 }
2368 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2369
cpufreq_driver_target(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)2370 int cpufreq_driver_target(struct cpufreq_policy *policy,
2371 unsigned int target_freq,
2372 unsigned int relation)
2373 {
2374 int ret;
2375
2376 down_write(&policy->rwsem);
2377
2378 ret = __cpufreq_driver_target(policy, target_freq, relation);
2379
2380 up_write(&policy->rwsem);
2381
2382 return ret;
2383 }
2384 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2385
cpufreq_fallback_governor(void)2386 __weak struct cpufreq_governor *cpufreq_fallback_governor(void)
2387 {
2388 return NULL;
2389 }
2390
cpufreq_init_governor(struct cpufreq_policy * policy)2391 static int cpufreq_init_governor(struct cpufreq_policy *policy)
2392 {
2393 int ret;
2394
2395 /* Don't start any governor operations if we are entering suspend */
2396 if (cpufreq_suspended)
2397 return 0;
2398 /*
2399 * Governor might not be initiated here if ACPI _PPC changed
2400 * notification happened, so check it.
2401 */
2402 if (!policy->governor)
2403 return -EINVAL;
2404
2405 /* Platform doesn't want dynamic frequency switching ? */
2406 if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING &&
2407 cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
2408 struct cpufreq_governor *gov = cpufreq_fallback_governor();
2409
2410 if (gov) {
2411 pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
2412 policy->governor->name, gov->name);
2413 policy->governor = gov;
2414 } else {
2415 return -EINVAL;
2416 }
2417 }
2418
2419 if (!try_module_get(policy->governor->owner))
2420 return -EINVAL;
2421
2422 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2423
2424 if (policy->governor->init) {
2425 ret = policy->governor->init(policy);
2426 if (ret) {
2427 module_put(policy->governor->owner);
2428 return ret;
2429 }
2430 }
2431
2432 policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET);
2433
2434 return 0;
2435 }
2436
cpufreq_exit_governor(struct cpufreq_policy * policy)2437 static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2438 {
2439 if (cpufreq_suspended || !policy->governor)
2440 return;
2441
2442 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2443
2444 if (policy->governor->exit)
2445 policy->governor->exit(policy);
2446
2447 module_put(policy->governor->owner);
2448 }
2449
cpufreq_start_governor(struct cpufreq_policy * policy)2450 int cpufreq_start_governor(struct cpufreq_policy *policy)
2451 {
2452 int ret;
2453
2454 if (cpufreq_suspended)
2455 return 0;
2456
2457 if (!policy->governor)
2458 return -EINVAL;
2459
2460 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2461
2462 if (cpufreq_driver->get)
2463 cpufreq_verify_current_freq(policy, false);
2464
2465 if (policy->governor->start) {
2466 ret = policy->governor->start(policy);
2467 if (ret)
2468 return ret;
2469 }
2470
2471 if (policy->governor->limits)
2472 policy->governor->limits(policy);
2473
2474 return 0;
2475 }
2476
cpufreq_stop_governor(struct cpufreq_policy * policy)2477 void cpufreq_stop_governor(struct cpufreq_policy *policy)
2478 {
2479 if (cpufreq_suspended || !policy->governor)
2480 return;
2481
2482 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2483
2484 if (policy->governor->stop)
2485 policy->governor->stop(policy);
2486 }
2487
cpufreq_governor_limits(struct cpufreq_policy * policy)2488 static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2489 {
2490 if (cpufreq_suspended || !policy->governor)
2491 return;
2492
2493 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2494
2495 if (policy->governor->limits)
2496 policy->governor->limits(policy);
2497 }
2498
cpufreq_register_governor(struct cpufreq_governor * governor)2499 int cpufreq_register_governor(struct cpufreq_governor *governor)
2500 {
2501 int err;
2502
2503 if (!governor)
2504 return -EINVAL;
2505
2506 if (cpufreq_disabled())
2507 return -ENODEV;
2508
2509 mutex_lock(&cpufreq_governor_mutex);
2510
2511 err = -EBUSY;
2512 if (!find_governor(governor->name)) {
2513 err = 0;
2514 list_add(&governor->governor_list, &cpufreq_governor_list);
2515 }
2516
2517 mutex_unlock(&cpufreq_governor_mutex);
2518 return err;
2519 }
2520 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2521
cpufreq_unregister_governor(struct cpufreq_governor * governor)2522 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2523 {
2524 struct cpufreq_policy *policy;
2525 unsigned long flags;
2526
2527 if (!governor)
2528 return;
2529
2530 if (cpufreq_disabled())
2531 return;
2532
2533 /* clear last_governor for all inactive policies */
2534 read_lock_irqsave(&cpufreq_driver_lock, flags);
2535 for_each_inactive_policy(policy) {
2536 if (!strcmp(policy->last_governor, governor->name)) {
2537 policy->governor = NULL;
2538 strcpy(policy->last_governor, "\0");
2539 }
2540 }
2541 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2542
2543 mutex_lock(&cpufreq_governor_mutex);
2544 list_del(&governor->governor_list);
2545 mutex_unlock(&cpufreq_governor_mutex);
2546 }
2547 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2548
2549
2550 /*********************************************************************
2551 * POLICY INTERFACE *
2552 *********************************************************************/
2553
2554 /**
2555 * cpufreq_get_policy - get the current cpufreq_policy
2556 * @policy: struct cpufreq_policy into which the current cpufreq_policy
2557 * is written
2558 * @cpu: CPU to find the policy for
2559 *
2560 * Reads the current cpufreq policy.
2561 */
cpufreq_get_policy(struct cpufreq_policy * policy,unsigned int cpu)2562 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2563 {
2564 struct cpufreq_policy *cpu_policy;
2565 if (!policy)
2566 return -EINVAL;
2567
2568 cpu_policy = cpufreq_cpu_get(cpu);
2569 if (!cpu_policy)
2570 return -EINVAL;
2571
2572 memcpy(policy, cpu_policy, sizeof(*policy));
2573
2574 cpufreq_cpu_put(cpu_policy);
2575 return 0;
2576 }
2577 EXPORT_SYMBOL(cpufreq_get_policy);
2578
2579 /**
2580 * cpufreq_set_policy - Modify cpufreq policy parameters.
2581 * @policy: Policy object to modify.
2582 * @new_gov: Policy governor pointer.
2583 * @new_pol: Policy value (for drivers with built-in governors).
2584 *
2585 * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency
2586 * limits to be set for the policy, update @policy with the verified limits
2587 * values and either invoke the driver's ->setpolicy() callback (if present) or
2588 * carry out a governor update for @policy. That is, run the current governor's
2589 * ->limits() callback (if @new_gov points to the same object as the one in
2590 * @policy) or replace the governor for @policy with @new_gov.
2591 *
2592 * The cpuinfo part of @policy is not updated by this function.
2593 */
cpufreq_set_policy(struct cpufreq_policy * policy,struct cpufreq_governor * new_gov,unsigned int new_pol)2594 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2595 struct cpufreq_governor *new_gov,
2596 unsigned int new_pol)
2597 {
2598 struct cpufreq_policy_data new_data;
2599 struct cpufreq_governor *old_gov;
2600 int ret;
2601
2602 memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2603 new_data.freq_table = policy->freq_table;
2604 new_data.cpu = policy->cpu;
2605 /*
2606 * PM QoS framework collects all the requests from users and provide us
2607 * the final aggregated value here.
2608 */
2609 new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN);
2610 new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX);
2611
2612 pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2613 new_data.cpu, new_data.min, new_data.max);
2614
2615 /*
2616 * Verify that the CPU speed can be set within these limits and make sure
2617 * that min <= max.
2618 */
2619 ret = cpufreq_driver->verify(&new_data);
2620 if (ret)
2621 return ret;
2622
2623 /*
2624 * Resolve policy min/max to available frequencies. It ensures
2625 * no frequency resolution will neither overshoot the requested maximum
2626 * nor undershoot the requested minimum.
2627 */
2628 policy->min = new_data.min;
2629 policy->max = new_data.max;
2630 policy->min = __resolve_freq(policy, policy->min, CPUFREQ_RELATION_L);
2631 policy->max = __resolve_freq(policy, policy->max, CPUFREQ_RELATION_H);
2632 trace_cpu_frequency_limits(policy);
2633
2634 policy->cached_target_freq = UINT_MAX;
2635
2636 pr_debug("new min and max freqs are %u - %u kHz\n",
2637 policy->min, policy->max);
2638
2639 if (cpufreq_driver->setpolicy) {
2640 policy->policy = new_pol;
2641 pr_debug("setting range\n");
2642 return cpufreq_driver->setpolicy(policy);
2643 }
2644
2645 if (new_gov == policy->governor) {
2646 pr_debug("governor limits update\n");
2647 cpufreq_governor_limits(policy);
2648 return 0;
2649 }
2650
2651 pr_debug("governor switch\n");
2652
2653 /* save old, working values */
2654 old_gov = policy->governor;
2655 /* end old governor */
2656 if (old_gov) {
2657 cpufreq_stop_governor(policy);
2658 cpufreq_exit_governor(policy);
2659 }
2660
2661 /* start new governor */
2662 policy->governor = new_gov;
2663 ret = cpufreq_init_governor(policy);
2664 if (!ret) {
2665 ret = cpufreq_start_governor(policy);
2666 if (!ret) {
2667 pr_debug("governor change\n");
2668 sched_cpufreq_governor_change(policy, old_gov);
2669 return 0;
2670 }
2671 cpufreq_exit_governor(policy);
2672 }
2673
2674 /* new governor failed, so re-start old one */
2675 pr_debug("starting governor %s failed\n", policy->governor->name);
2676 if (old_gov) {
2677 policy->governor = old_gov;
2678 if (cpufreq_init_governor(policy))
2679 policy->governor = NULL;
2680 else
2681 cpufreq_start_governor(policy);
2682 }
2683
2684 return ret;
2685 }
2686
2687 /**
2688 * cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
2689 * @cpu: CPU to re-evaluate the policy for.
2690 *
2691 * Update the current frequency for the cpufreq policy of @cpu and use
2692 * cpufreq_set_policy() to re-apply the min and max limits, which triggers the
2693 * evaluation of policy notifiers and the cpufreq driver's ->verify() callback
2694 * for the policy in question, among other things.
2695 */
cpufreq_update_policy(unsigned int cpu)2696 void cpufreq_update_policy(unsigned int cpu)
2697 {
2698 struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
2699
2700 if (!policy)
2701 return;
2702
2703 /*
2704 * BIOS might change freq behind our back
2705 * -> ask driver for current freq and notify governors about a change
2706 */
2707 if (cpufreq_driver->get && has_target() &&
2708 (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false))))
2709 goto unlock;
2710
2711 refresh_frequency_limits(policy);
2712
2713 unlock:
2714 cpufreq_cpu_release(policy);
2715 }
2716 EXPORT_SYMBOL(cpufreq_update_policy);
2717
2718 /**
2719 * cpufreq_update_limits - Update policy limits for a given CPU.
2720 * @cpu: CPU to update the policy limits for.
2721 *
2722 * Invoke the driver's ->update_limits callback if present or call
2723 * cpufreq_update_policy() for @cpu.
2724 */
cpufreq_update_limits(unsigned int cpu)2725 void cpufreq_update_limits(unsigned int cpu)
2726 {
2727 if (cpufreq_driver->update_limits)
2728 cpufreq_driver->update_limits(cpu);
2729 else
2730 cpufreq_update_policy(cpu);
2731 }
2732 EXPORT_SYMBOL_GPL(cpufreq_update_limits);
2733
2734 /*********************************************************************
2735 * BOOST *
2736 *********************************************************************/
cpufreq_boost_set_sw(struct cpufreq_policy * policy,int state)2737 static int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
2738 {
2739 int ret;
2740
2741 if (!policy->freq_table)
2742 return -ENXIO;
2743
2744 ret = cpufreq_frequency_table_cpuinfo(policy, policy->freq_table);
2745 if (ret) {
2746 pr_err("%s: Policy frequency update failed\n", __func__);
2747 return ret;
2748 }
2749
2750 ret = freq_qos_update_request(policy->max_freq_req, policy->max);
2751 if (ret < 0)
2752 return ret;
2753
2754 return 0;
2755 }
2756
cpufreq_boost_trigger_state(int state)2757 int cpufreq_boost_trigger_state(int state)
2758 {
2759 struct cpufreq_policy *policy;
2760 unsigned long flags;
2761 int ret = 0;
2762
2763 if (cpufreq_driver->boost_enabled == state)
2764 return 0;
2765
2766 write_lock_irqsave(&cpufreq_driver_lock, flags);
2767 cpufreq_driver->boost_enabled = state;
2768 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2769
2770 cpus_read_lock();
2771 for_each_active_policy(policy) {
2772 policy->boost_enabled = state;
2773 ret = cpufreq_driver->set_boost(policy, state);
2774 if (ret) {
2775 policy->boost_enabled = !policy->boost_enabled;
2776 goto err_reset_state;
2777 }
2778 }
2779 cpus_read_unlock();
2780
2781 return 0;
2782
2783 err_reset_state:
2784 cpus_read_unlock();
2785
2786 write_lock_irqsave(&cpufreq_driver_lock, flags);
2787 cpufreq_driver->boost_enabled = !state;
2788 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2789
2790 pr_err("%s: Cannot %s BOOST\n",
2791 __func__, state ? "enable" : "disable");
2792
2793 return ret;
2794 }
2795
cpufreq_boost_supported(void)2796 static bool cpufreq_boost_supported(void)
2797 {
2798 return cpufreq_driver->set_boost;
2799 }
2800
create_boost_sysfs_file(void)2801 static int create_boost_sysfs_file(void)
2802 {
2803 int ret;
2804
2805 ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2806 if (ret)
2807 pr_err("%s: cannot register global BOOST sysfs file\n",
2808 __func__);
2809
2810 return ret;
2811 }
2812
remove_boost_sysfs_file(void)2813 static void remove_boost_sysfs_file(void)
2814 {
2815 if (cpufreq_boost_supported())
2816 sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2817 }
2818
cpufreq_enable_boost_support(void)2819 int cpufreq_enable_boost_support(void)
2820 {
2821 if (!cpufreq_driver)
2822 return -EINVAL;
2823
2824 if (cpufreq_boost_supported())
2825 return 0;
2826
2827 cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2828
2829 /* This will get removed on driver unregister */
2830 return create_boost_sysfs_file();
2831 }
2832 EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2833
cpufreq_boost_enabled(void)2834 int cpufreq_boost_enabled(void)
2835 {
2836 return cpufreq_driver->boost_enabled;
2837 }
2838 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2839
2840 /*********************************************************************
2841 * REGISTER / UNREGISTER CPUFREQ DRIVER *
2842 *********************************************************************/
2843 static enum cpuhp_state hp_online;
2844
cpuhp_cpufreq_online(unsigned int cpu)2845 static int cpuhp_cpufreq_online(unsigned int cpu)
2846 {
2847 cpufreq_online(cpu);
2848
2849 return 0;
2850 }
2851
cpuhp_cpufreq_offline(unsigned int cpu)2852 static int cpuhp_cpufreq_offline(unsigned int cpu)
2853 {
2854 cpufreq_offline(cpu);
2855
2856 return 0;
2857 }
2858
2859 /**
2860 * cpufreq_register_driver - register a CPU Frequency driver
2861 * @driver_data: A struct cpufreq_driver containing the values#
2862 * submitted by the CPU Frequency driver.
2863 *
2864 * Registers a CPU Frequency driver to this core code. This code
2865 * returns zero on success, -EEXIST when another driver got here first
2866 * (and isn't unregistered in the meantime).
2867 *
2868 */
cpufreq_register_driver(struct cpufreq_driver * driver_data)2869 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2870 {
2871 unsigned long flags;
2872 int ret;
2873
2874 if (cpufreq_disabled())
2875 return -ENODEV;
2876
2877 /*
2878 * The cpufreq core depends heavily on the availability of device
2879 * structure, make sure they are available before proceeding further.
2880 */
2881 if (!get_cpu_device(0))
2882 return -EPROBE_DEFER;
2883
2884 if (!driver_data || !driver_data->verify || !driver_data->init ||
2885 !(driver_data->setpolicy || driver_data->target_index ||
2886 driver_data->target) ||
2887 (driver_data->setpolicy && (driver_data->target_index ||
2888 driver_data->target)) ||
2889 (!driver_data->get_intermediate != !driver_data->target_intermediate) ||
2890 (!driver_data->online != !driver_data->offline) ||
2891 (driver_data->adjust_perf && !driver_data->fast_switch))
2892 return -EINVAL;
2893
2894 pr_debug("trying to register driver %s\n", driver_data->name);
2895
2896 /* Protect against concurrent CPU online/offline. */
2897 cpus_read_lock();
2898
2899 write_lock_irqsave(&cpufreq_driver_lock, flags);
2900 if (cpufreq_driver) {
2901 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2902 ret = -EEXIST;
2903 goto out;
2904 }
2905 cpufreq_driver = driver_data;
2906 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2907
2908 /*
2909 * Mark support for the scheduler's frequency invariance engine for
2910 * drivers that implement target(), target_index() or fast_switch().
2911 */
2912 if (!cpufreq_driver->setpolicy) {
2913 static_branch_enable_cpuslocked(&cpufreq_freq_invariance);
2914 pr_debug("supports frequency invariance");
2915 }
2916
2917 if (driver_data->setpolicy)
2918 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2919
2920 if (cpufreq_boost_supported()) {
2921 ret = create_boost_sysfs_file();
2922 if (ret)
2923 goto err_null_driver;
2924 }
2925
2926 ret = subsys_interface_register(&cpufreq_interface);
2927 if (ret)
2928 goto err_boost_unreg;
2929
2930 if (unlikely(list_empty(&cpufreq_policy_list))) {
2931 /* if all ->init() calls failed, unregister */
2932 ret = -ENODEV;
2933 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2934 driver_data->name);
2935 goto err_if_unreg;
2936 }
2937
2938 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
2939 "cpufreq:online",
2940 cpuhp_cpufreq_online,
2941 cpuhp_cpufreq_offline);
2942 if (ret < 0)
2943 goto err_if_unreg;
2944 hp_online = ret;
2945 ret = 0;
2946
2947 pr_debug("driver %s up and running\n", driver_data->name);
2948 goto out;
2949
2950 err_if_unreg:
2951 subsys_interface_unregister(&cpufreq_interface);
2952 err_boost_unreg:
2953 remove_boost_sysfs_file();
2954 err_null_driver:
2955 write_lock_irqsave(&cpufreq_driver_lock, flags);
2956 cpufreq_driver = NULL;
2957 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2958 out:
2959 cpus_read_unlock();
2960 return ret;
2961 }
2962 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2963
2964 /*
2965 * cpufreq_unregister_driver - unregister the current CPUFreq driver
2966 *
2967 * Unregister the current CPUFreq driver. Only call this if you have
2968 * the right to do so, i.e. if you have succeeded in initialising before!
2969 * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2970 * currently not initialised.
2971 */
cpufreq_unregister_driver(struct cpufreq_driver * driver)2972 void cpufreq_unregister_driver(struct cpufreq_driver *driver)
2973 {
2974 unsigned long flags;
2975
2976 if (WARN_ON(!cpufreq_driver || (driver != cpufreq_driver)))
2977 return;
2978
2979 pr_debug("unregistering driver %s\n", driver->name);
2980
2981 /* Protect against concurrent cpu hotplug */
2982 cpus_read_lock();
2983 subsys_interface_unregister(&cpufreq_interface);
2984 remove_boost_sysfs_file();
2985 static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
2986 cpuhp_remove_state_nocalls_cpuslocked(hp_online);
2987
2988 write_lock_irqsave(&cpufreq_driver_lock, flags);
2989
2990 cpufreq_driver = NULL;
2991
2992 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2993 cpus_read_unlock();
2994 }
2995 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2996
cpufreq_core_init(void)2997 static int __init cpufreq_core_init(void)
2998 {
2999 struct cpufreq_governor *gov = cpufreq_default_governor();
3000 struct device *dev_root;
3001
3002 if (cpufreq_disabled())
3003 return -ENODEV;
3004
3005 dev_root = bus_get_dev_root(&cpu_subsys);
3006 if (dev_root) {
3007 cpufreq_global_kobject = kobject_create_and_add("cpufreq", &dev_root->kobj);
3008 put_device(dev_root);
3009 }
3010 BUG_ON(!cpufreq_global_kobject);
3011
3012 if (!strlen(default_governor))
3013 strncpy(default_governor, gov->name, CPUFREQ_NAME_LEN);
3014
3015 return 0;
3016 }
3017 module_param(off, int, 0444);
3018 module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444);
3019 core_initcall(cpufreq_core_init);
3020