xref: /openbmc/linux/drivers/cpufreq/cpufreq.c (revision 9ac8d3fb)
1 /*
2  *  linux/drivers/cpufreq/cpufreq.c
3  *
4  *  Copyright (C) 2001 Russell King
5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
6  *
7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
8  *	Added handling for CPU hotplug
9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
10  *	Fix handling for CPU hotplug -- affected CPUs
11  *
12  * This program is free software; you can redistribute it and/or modify
13  * it under the terms of the GNU General Public License version 2 as
14  * published by the Free Software Foundation.
15  *
16  */
17 
18 #include <linux/kernel.h>
19 #include <linux/module.h>
20 #include <linux/init.h>
21 #include <linux/notifier.h>
22 #include <linux/cpufreq.h>
23 #include <linux/delay.h>
24 #include <linux/interrupt.h>
25 #include <linux/spinlock.h>
26 #include <linux/device.h>
27 #include <linux/slab.h>
28 #include <linux/cpu.h>
29 #include <linux/completion.h>
30 #include <linux/mutex.h>
31 
32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
33 						"cpufreq-core", msg)
34 
35 /**
36  * The "cpufreq driver" - the arch- or hardware-dependent low
37  * level driver of CPUFreq support, and its spinlock. This lock
38  * also protects the cpufreq_cpu_data array.
39  */
40 static struct cpufreq_driver *cpufreq_driver;
41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
42 #ifdef CONFIG_HOTPLUG_CPU
43 /* This one keeps track of the previously set governor of a removed CPU */
44 static DEFINE_PER_CPU(struct cpufreq_governor *, cpufreq_cpu_governor);
45 #endif
46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
47 
48 /*
49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
50  * all cpufreq/hotplug/workqueue/etc related lock issues.
51  *
52  * The rules for this semaphore:
53  * - Any routine that wants to read from the policy structure will
54  *   do a down_read on this semaphore.
55  * - Any routine that will write to the policy structure and/or may take away
56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
57  *   mode before doing so.
58  *
59  * Additional rules:
60  * - All holders of the lock should check to make sure that the CPU they
61  *   are concerned with are online after they get the lock.
62  * - Governor routines that can be called in cpufreq hotplug path should not
63  *   take this sem as top level hotplug notifier handler takes this.
64  */
65 static DEFINE_PER_CPU(int, policy_cpu);
66 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
67 
68 #define lock_policy_rwsem(mode, cpu)					\
69 int lock_policy_rwsem_##mode						\
70 (int cpu)								\
71 {									\
72 	int policy_cpu = per_cpu(policy_cpu, cpu);			\
73 	BUG_ON(policy_cpu == -1);					\
74 	down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));		\
75 	if (unlikely(!cpu_online(cpu))) {				\
76 		up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));	\
77 		return -1;						\
78 	}								\
79 									\
80 	return 0;							\
81 }
82 
83 lock_policy_rwsem(read, cpu);
84 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
85 
86 lock_policy_rwsem(write, cpu);
87 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
88 
89 void unlock_policy_rwsem_read(int cpu)
90 {
91 	int policy_cpu = per_cpu(policy_cpu, cpu);
92 	BUG_ON(policy_cpu == -1);
93 	up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
94 }
95 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
96 
97 void unlock_policy_rwsem_write(int cpu)
98 {
99 	int policy_cpu = per_cpu(policy_cpu, cpu);
100 	BUG_ON(policy_cpu == -1);
101 	up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
102 }
103 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
104 
105 
106 /* internal prototypes */
107 static int __cpufreq_governor(struct cpufreq_policy *policy, unsigned int event);
108 static unsigned int __cpufreq_get(unsigned int cpu);
109 static void handle_update(struct work_struct *work);
110 
111 /**
112  * Two notifier lists: the "policy" list is involved in the
113  * validation process for a new CPU frequency policy; the
114  * "transition" list for kernel code that needs to handle
115  * changes to devices when the CPU clock speed changes.
116  * The mutex locks both lists.
117  */
118 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
119 static struct srcu_notifier_head cpufreq_transition_notifier_list;
120 
121 static bool init_cpufreq_transition_notifier_list_called;
122 static int __init init_cpufreq_transition_notifier_list(void)
123 {
124 	srcu_init_notifier_head(&cpufreq_transition_notifier_list);
125 	init_cpufreq_transition_notifier_list_called = true;
126 	return 0;
127 }
128 pure_initcall(init_cpufreq_transition_notifier_list);
129 
130 static LIST_HEAD(cpufreq_governor_list);
131 static DEFINE_MUTEX (cpufreq_governor_mutex);
132 
133 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
134 {
135 	struct cpufreq_policy *data;
136 	unsigned long flags;
137 
138 	if (cpu >= nr_cpu_ids)
139 		goto err_out;
140 
141 	/* get the cpufreq driver */
142 	spin_lock_irqsave(&cpufreq_driver_lock, flags);
143 
144 	if (!cpufreq_driver)
145 		goto err_out_unlock;
146 
147 	if (!try_module_get(cpufreq_driver->owner))
148 		goto err_out_unlock;
149 
150 
151 	/* get the CPU */
152 	data = per_cpu(cpufreq_cpu_data, cpu);
153 
154 	if (!data)
155 		goto err_out_put_module;
156 
157 	if (!kobject_get(&data->kobj))
158 		goto err_out_put_module;
159 
160 	spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
161 	return data;
162 
163 err_out_put_module:
164 	module_put(cpufreq_driver->owner);
165 err_out_unlock:
166 	spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
167 err_out:
168 	return NULL;
169 }
170 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
171 
172 
173 void cpufreq_cpu_put(struct cpufreq_policy *data)
174 {
175 	kobject_put(&data->kobj);
176 	module_put(cpufreq_driver->owner);
177 }
178 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
179 
180 
181 /*********************************************************************
182  *                     UNIFIED DEBUG HELPERS                         *
183  *********************************************************************/
184 #ifdef CONFIG_CPU_FREQ_DEBUG
185 
186 /* what part(s) of the CPUfreq subsystem are debugged? */
187 static unsigned int debug;
188 
189 /* is the debug output ratelimit'ed using printk_ratelimit? User can
190  * set or modify this value.
191  */
192 static unsigned int debug_ratelimit = 1;
193 
194 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
195  * loading of a cpufreq driver, temporarily disabled when a new policy
196  * is set, and disabled upon cpufreq driver removal
197  */
198 static unsigned int disable_ratelimit = 1;
199 static DEFINE_SPINLOCK(disable_ratelimit_lock);
200 
201 static void cpufreq_debug_enable_ratelimit(void)
202 {
203 	unsigned long flags;
204 
205 	spin_lock_irqsave(&disable_ratelimit_lock, flags);
206 	if (disable_ratelimit)
207 		disable_ratelimit--;
208 	spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
209 }
210 
211 static void cpufreq_debug_disable_ratelimit(void)
212 {
213 	unsigned long flags;
214 
215 	spin_lock_irqsave(&disable_ratelimit_lock, flags);
216 	disable_ratelimit++;
217 	spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
218 }
219 
220 void cpufreq_debug_printk(unsigned int type, const char *prefix,
221 			const char *fmt, ...)
222 {
223 	char s[256];
224 	va_list args;
225 	unsigned int len;
226 	unsigned long flags;
227 
228 	WARN_ON(!prefix);
229 	if (type & debug) {
230 		spin_lock_irqsave(&disable_ratelimit_lock, flags);
231 		if (!disable_ratelimit && debug_ratelimit
232 					&& !printk_ratelimit()) {
233 			spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
234 			return;
235 		}
236 		spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
237 
238 		len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
239 
240 		va_start(args, fmt);
241 		len += vsnprintf(&s[len], (256 - len), fmt, args);
242 		va_end(args);
243 
244 		printk(s);
245 
246 		WARN_ON(len < 5);
247 	}
248 }
249 EXPORT_SYMBOL(cpufreq_debug_printk);
250 
251 
252 module_param(debug, uint, 0644);
253 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
254 			" 2 to debug drivers, and 4 to debug governors.");
255 
256 module_param(debug_ratelimit, uint, 0644);
257 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
258 					" set to 0 to disable ratelimiting.");
259 
260 #else /* !CONFIG_CPU_FREQ_DEBUG */
261 
262 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
263 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
264 
265 #endif /* CONFIG_CPU_FREQ_DEBUG */
266 
267 
268 /*********************************************************************
269  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
270  *********************************************************************/
271 
272 /**
273  * adjust_jiffies - adjust the system "loops_per_jiffy"
274  *
275  * This function alters the system "loops_per_jiffy" for the clock
276  * speed change. Note that loops_per_jiffy cannot be updated on SMP
277  * systems as each CPU might be scaled differently. So, use the arch
278  * per-CPU loops_per_jiffy value wherever possible.
279  */
280 #ifndef CONFIG_SMP
281 static unsigned long l_p_j_ref;
282 static unsigned int  l_p_j_ref_freq;
283 
284 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
285 {
286 	if (ci->flags & CPUFREQ_CONST_LOOPS)
287 		return;
288 
289 	if (!l_p_j_ref_freq) {
290 		l_p_j_ref = loops_per_jiffy;
291 		l_p_j_ref_freq = ci->old;
292 		dprintk("saving %lu as reference value for loops_per_jiffy; "
293 			"freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
294 	}
295 	if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
296 	    (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
297 	    (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
298 		loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
299 								ci->new);
300 		dprintk("scaling loops_per_jiffy to %lu "
301 			"for frequency %u kHz\n", loops_per_jiffy, ci->new);
302 	}
303 }
304 #else
305 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
306 {
307 	return;
308 }
309 #endif
310 
311 
312 /**
313  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
314  * on frequency transition.
315  *
316  * This function calls the transition notifiers and the "adjust_jiffies"
317  * function. It is called twice on all CPU frequency changes that have
318  * external effects.
319  */
320 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
321 {
322 	struct cpufreq_policy *policy;
323 
324 	BUG_ON(irqs_disabled());
325 
326 	freqs->flags = cpufreq_driver->flags;
327 	dprintk("notification %u of frequency transition to %u kHz\n",
328 		state, freqs->new);
329 
330 	policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
331 	switch (state) {
332 
333 	case CPUFREQ_PRECHANGE:
334 		/* detect if the driver reported a value as "old frequency"
335 		 * which is not equal to what the cpufreq core thinks is
336 		 * "old frequency".
337 		 */
338 		if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
339 			if ((policy) && (policy->cpu == freqs->cpu) &&
340 			    (policy->cur) && (policy->cur != freqs->old)) {
341 				dprintk("Warning: CPU frequency is"
342 					" %u, cpufreq assumed %u kHz.\n",
343 					freqs->old, policy->cur);
344 				freqs->old = policy->cur;
345 			}
346 		}
347 		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
348 				CPUFREQ_PRECHANGE, freqs);
349 		adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
350 		break;
351 
352 	case CPUFREQ_POSTCHANGE:
353 		adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
354 		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
355 				CPUFREQ_POSTCHANGE, freqs);
356 		if (likely(policy) && likely(policy->cpu == freqs->cpu))
357 			policy->cur = freqs->new;
358 		break;
359 	}
360 }
361 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
362 
363 
364 
365 /*********************************************************************
366  *                          SYSFS INTERFACE                          *
367  *********************************************************************/
368 
369 static struct cpufreq_governor *__find_governor(const char *str_governor)
370 {
371 	struct cpufreq_governor *t;
372 
373 	list_for_each_entry(t, &cpufreq_governor_list, governor_list)
374 		if (!strnicmp(str_governor,t->name,CPUFREQ_NAME_LEN))
375 			return t;
376 
377 	return NULL;
378 }
379 
380 /**
381  * cpufreq_parse_governor - parse a governor string
382  */
383 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
384 				struct cpufreq_governor **governor)
385 {
386 	int err = -EINVAL;
387 
388 	if (!cpufreq_driver)
389 		goto out;
390 
391 	if (cpufreq_driver->setpolicy) {
392 		if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
393 			*policy = CPUFREQ_POLICY_PERFORMANCE;
394 			err = 0;
395 		} else if (!strnicmp(str_governor, "powersave",
396 						CPUFREQ_NAME_LEN)) {
397 			*policy = CPUFREQ_POLICY_POWERSAVE;
398 			err = 0;
399 		}
400 	} else if (cpufreq_driver->target) {
401 		struct cpufreq_governor *t;
402 
403 		mutex_lock(&cpufreq_governor_mutex);
404 
405 		t = __find_governor(str_governor);
406 
407 		if (t == NULL) {
408 			char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
409 								str_governor);
410 
411 			if (name) {
412 				int ret;
413 
414 				mutex_unlock(&cpufreq_governor_mutex);
415 				ret = request_module("%s", name);
416 				mutex_lock(&cpufreq_governor_mutex);
417 
418 				if (ret == 0)
419 					t = __find_governor(str_governor);
420 			}
421 
422 			kfree(name);
423 		}
424 
425 		if (t != NULL) {
426 			*governor = t;
427 			err = 0;
428 		}
429 
430 		mutex_unlock(&cpufreq_governor_mutex);
431 	}
432   out:
433 	return err;
434 }
435 
436 
437 /* drivers/base/cpu.c */
438 extern struct sysdev_class cpu_sysdev_class;
439 
440 
441 /**
442  * cpufreq_per_cpu_attr_read() / show_##file_name() -
443  * print out cpufreq information
444  *
445  * Write out information from cpufreq_driver->policy[cpu]; object must be
446  * "unsigned int".
447  */
448 
449 #define show_one(file_name, object)			\
450 static ssize_t show_##file_name				\
451 (struct cpufreq_policy *policy, char *buf)		\
452 {							\
453 	return sprintf (buf, "%u\n", policy->object);	\
454 }
455 
456 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
457 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
458 show_one(scaling_min_freq, min);
459 show_one(scaling_max_freq, max);
460 show_one(scaling_cur_freq, cur);
461 
462 static int __cpufreq_set_policy(struct cpufreq_policy *data,
463 				struct cpufreq_policy *policy);
464 
465 /**
466  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
467  */
468 #define store_one(file_name, object)			\
469 static ssize_t store_##file_name					\
470 (struct cpufreq_policy *policy, const char *buf, size_t count)		\
471 {									\
472 	unsigned int ret = -EINVAL;					\
473 	struct cpufreq_policy new_policy;				\
474 									\
475 	ret = cpufreq_get_policy(&new_policy, policy->cpu);		\
476 	if (ret)							\
477 		return -EINVAL;						\
478 									\
479 	ret = sscanf (buf, "%u", &new_policy.object);			\
480 	if (ret != 1)							\
481 		return -EINVAL;						\
482 									\
483 	ret = __cpufreq_set_policy(policy, &new_policy);		\
484 	policy->user_policy.object = policy->object;			\
485 									\
486 	return ret ? ret : count;					\
487 }
488 
489 store_one(scaling_min_freq,min);
490 store_one(scaling_max_freq,max);
491 
492 /**
493  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
494  */
495 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
496 					char *buf)
497 {
498 	unsigned int cur_freq = __cpufreq_get(policy->cpu);
499 	if (!cur_freq)
500 		return sprintf(buf, "<unknown>");
501 	return sprintf(buf, "%u\n", cur_freq);
502 }
503 
504 
505 /**
506  * show_scaling_governor - show the current policy for the specified CPU
507  */
508 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
509 {
510 	if(policy->policy == CPUFREQ_POLICY_POWERSAVE)
511 		return sprintf(buf, "powersave\n");
512 	else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
513 		return sprintf(buf, "performance\n");
514 	else if (policy->governor)
515 		return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", policy->governor->name);
516 	return -EINVAL;
517 }
518 
519 
520 /**
521  * store_scaling_governor - store policy for the specified CPU
522  */
523 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
524 					const char *buf, size_t count)
525 {
526 	unsigned int ret = -EINVAL;
527 	char	str_governor[16];
528 	struct cpufreq_policy new_policy;
529 
530 	ret = cpufreq_get_policy(&new_policy, policy->cpu);
531 	if (ret)
532 		return ret;
533 
534 	ret = sscanf (buf, "%15s", str_governor);
535 	if (ret != 1)
536 		return -EINVAL;
537 
538 	if (cpufreq_parse_governor(str_governor, &new_policy.policy,
539 						&new_policy.governor))
540 		return -EINVAL;
541 
542 	/* Do not use cpufreq_set_policy here or the user_policy.max
543 	   will be wrongly overridden */
544 	ret = __cpufreq_set_policy(policy, &new_policy);
545 
546 	policy->user_policy.policy = policy->policy;
547 	policy->user_policy.governor = policy->governor;
548 
549 	if (ret)
550 		return ret;
551 	else
552 		return count;
553 }
554 
555 /**
556  * show_scaling_driver - show the cpufreq driver currently loaded
557  */
558 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
559 {
560 	return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
561 }
562 
563 /**
564  * show_scaling_available_governors - show the available CPUfreq governors
565  */
566 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
567 						char *buf)
568 {
569 	ssize_t i = 0;
570 	struct cpufreq_governor *t;
571 
572 	if (!cpufreq_driver->target) {
573 		i += sprintf(buf, "performance powersave");
574 		goto out;
575 	}
576 
577 	list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
578 		if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char)) - (CPUFREQ_NAME_LEN + 2)))
579 			goto out;
580 		i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
581 	}
582 out:
583 	i += sprintf(&buf[i], "\n");
584 	return i;
585 }
586 
587 static ssize_t show_cpus(cpumask_t mask, char *buf)
588 {
589 	ssize_t i = 0;
590 	unsigned int cpu;
591 
592 	for_each_cpu_mask_nr(cpu, mask) {
593 		if (i)
594 			i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
595 		i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
596 		if (i >= (PAGE_SIZE - 5))
597 		    break;
598 	}
599 	i += sprintf(&buf[i], "\n");
600 	return i;
601 }
602 
603 /**
604  * show_related_cpus - show the CPUs affected by each transition even if
605  * hw coordination is in use
606  */
607 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
608 {
609 	if (cpus_empty(policy->related_cpus))
610 		return show_cpus(policy->cpus, buf);
611 	return show_cpus(policy->related_cpus, buf);
612 }
613 
614 /**
615  * show_affected_cpus - show the CPUs affected by each transition
616  */
617 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
618 {
619 	return show_cpus(policy->cpus, buf);
620 }
621 
622 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
623 					const char *buf, size_t count)
624 {
625 	unsigned int freq = 0;
626 	unsigned int ret;
627 
628 	if (!policy->governor || !policy->governor->store_setspeed)
629 		return -EINVAL;
630 
631 	ret = sscanf(buf, "%u", &freq);
632 	if (ret != 1)
633 		return -EINVAL;
634 
635 	policy->governor->store_setspeed(policy, freq);
636 
637 	return count;
638 }
639 
640 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
641 {
642 	if (!policy->governor || !policy->governor->show_setspeed)
643 		return sprintf(buf, "<unsupported>\n");
644 
645 	return policy->governor->show_setspeed(policy, buf);
646 }
647 
648 #define define_one_ro(_name) \
649 static struct freq_attr _name = \
650 __ATTR(_name, 0444, show_##_name, NULL)
651 
652 #define define_one_ro0400(_name) \
653 static struct freq_attr _name = \
654 __ATTR(_name, 0400, show_##_name, NULL)
655 
656 #define define_one_rw(_name) \
657 static struct freq_attr _name = \
658 __ATTR(_name, 0644, show_##_name, store_##_name)
659 
660 define_one_ro0400(cpuinfo_cur_freq);
661 define_one_ro(cpuinfo_min_freq);
662 define_one_ro(cpuinfo_max_freq);
663 define_one_ro(scaling_available_governors);
664 define_one_ro(scaling_driver);
665 define_one_ro(scaling_cur_freq);
666 define_one_ro(related_cpus);
667 define_one_ro(affected_cpus);
668 define_one_rw(scaling_min_freq);
669 define_one_rw(scaling_max_freq);
670 define_one_rw(scaling_governor);
671 define_one_rw(scaling_setspeed);
672 
673 static struct attribute *default_attrs[] = {
674 	&cpuinfo_min_freq.attr,
675 	&cpuinfo_max_freq.attr,
676 	&scaling_min_freq.attr,
677 	&scaling_max_freq.attr,
678 	&affected_cpus.attr,
679 	&related_cpus.attr,
680 	&scaling_governor.attr,
681 	&scaling_driver.attr,
682 	&scaling_available_governors.attr,
683 	&scaling_setspeed.attr,
684 	NULL
685 };
686 
687 #define to_policy(k) container_of(k,struct cpufreq_policy,kobj)
688 #define to_attr(a) container_of(a,struct freq_attr,attr)
689 
690 static ssize_t show(struct kobject *kobj, struct attribute *attr ,char *buf)
691 {
692 	struct cpufreq_policy *policy = to_policy(kobj);
693 	struct freq_attr *fattr = to_attr(attr);
694 	ssize_t ret = -EINVAL;
695 	policy = cpufreq_cpu_get(policy->cpu);
696 	if (!policy)
697 		goto no_policy;
698 
699 	if (lock_policy_rwsem_read(policy->cpu) < 0)
700 		goto fail;
701 
702 	if (fattr->show)
703 		ret = fattr->show(policy, buf);
704 	else
705 		ret = -EIO;
706 
707 	unlock_policy_rwsem_read(policy->cpu);
708 fail:
709 	cpufreq_cpu_put(policy);
710 no_policy:
711 	return ret;
712 }
713 
714 static ssize_t store(struct kobject *kobj, struct attribute *attr,
715 		     const char *buf, size_t count)
716 {
717 	struct cpufreq_policy *policy = to_policy(kobj);
718 	struct freq_attr *fattr = to_attr(attr);
719 	ssize_t ret = -EINVAL;
720 	policy = cpufreq_cpu_get(policy->cpu);
721 	if (!policy)
722 		goto no_policy;
723 
724 	if (lock_policy_rwsem_write(policy->cpu) < 0)
725 		goto fail;
726 
727 	if (fattr->store)
728 		ret = fattr->store(policy, buf, count);
729 	else
730 		ret = -EIO;
731 
732 	unlock_policy_rwsem_write(policy->cpu);
733 fail:
734 	cpufreq_cpu_put(policy);
735 no_policy:
736 	return ret;
737 }
738 
739 static void cpufreq_sysfs_release(struct kobject *kobj)
740 {
741 	struct cpufreq_policy *policy = to_policy(kobj);
742 	dprintk("last reference is dropped\n");
743 	complete(&policy->kobj_unregister);
744 }
745 
746 static struct sysfs_ops sysfs_ops = {
747 	.show	= show,
748 	.store	= store,
749 };
750 
751 static struct kobj_type ktype_cpufreq = {
752 	.sysfs_ops	= &sysfs_ops,
753 	.default_attrs	= default_attrs,
754 	.release	= cpufreq_sysfs_release,
755 };
756 
757 
758 /**
759  * cpufreq_add_dev - add a CPU device
760  *
761  * Adds the cpufreq interface for a CPU device.
762  */
763 static int cpufreq_add_dev(struct sys_device *sys_dev)
764 {
765 	unsigned int cpu = sys_dev->id;
766 	int ret = 0;
767 	struct cpufreq_policy new_policy;
768 	struct cpufreq_policy *policy;
769 	struct freq_attr **drv_attr;
770 	struct sys_device *cpu_sys_dev;
771 	unsigned long flags;
772 	unsigned int j;
773 #ifdef CONFIG_SMP
774 	struct cpufreq_policy *managed_policy;
775 #endif
776 
777 	if (cpu_is_offline(cpu))
778 		return 0;
779 
780 	cpufreq_debug_disable_ratelimit();
781 	dprintk("adding CPU %u\n", cpu);
782 
783 #ifdef CONFIG_SMP
784 	/* check whether a different CPU already registered this
785 	 * CPU because it is in the same boat. */
786 	policy = cpufreq_cpu_get(cpu);
787 	if (unlikely(policy)) {
788 		cpufreq_cpu_put(policy);
789 		cpufreq_debug_enable_ratelimit();
790 		return 0;
791 	}
792 #endif
793 
794 	if (!try_module_get(cpufreq_driver->owner)) {
795 		ret = -EINVAL;
796 		goto module_out;
797 	}
798 
799 	policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
800 	if (!policy) {
801 		ret = -ENOMEM;
802 		goto nomem_out;
803 	}
804 
805 	policy->cpu = cpu;
806 	policy->cpus = cpumask_of_cpu(cpu);
807 
808 	/* Initially set CPU itself as the policy_cpu */
809 	per_cpu(policy_cpu, cpu) = cpu;
810 	lock_policy_rwsem_write(cpu);
811 
812 	init_completion(&policy->kobj_unregister);
813 	INIT_WORK(&policy->update, handle_update);
814 
815 	/* Set governor before ->init, so that driver could check it */
816 	policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
817 	/* call driver. From then on the cpufreq must be able
818 	 * to accept all calls to ->verify and ->setpolicy for this CPU
819 	 */
820 	ret = cpufreq_driver->init(policy);
821 	if (ret) {
822 		dprintk("initialization failed\n");
823 		goto err_out;
824 	}
825 	policy->user_policy.min = policy->cpuinfo.min_freq;
826 	policy->user_policy.max = policy->cpuinfo.max_freq;
827 
828 	blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
829 				     CPUFREQ_START, policy);
830 
831 #ifdef CONFIG_SMP
832 
833 #ifdef CONFIG_HOTPLUG_CPU
834 	if (per_cpu(cpufreq_cpu_governor, cpu)) {
835 		policy->governor = per_cpu(cpufreq_cpu_governor, cpu);
836 		dprintk("Restoring governor %s for cpu %d\n",
837 		       policy->governor->name, cpu);
838 	}
839 #endif
840 
841 	for_each_cpu_mask_nr(j, policy->cpus) {
842 		if (cpu == j)
843 			continue;
844 
845 		/* check for existing affected CPUs.  They may not be aware
846 		 * of it due to CPU Hotplug.
847 		 */
848 		managed_policy = cpufreq_cpu_get(j);		// FIXME: Where is this released?  What about error paths?
849 		if (unlikely(managed_policy)) {
850 
851 			/* Set proper policy_cpu */
852 			unlock_policy_rwsem_write(cpu);
853 			per_cpu(policy_cpu, cpu) = managed_policy->cpu;
854 
855 			if (lock_policy_rwsem_write(cpu) < 0)
856 				goto err_out_driver_exit;
857 
858 			spin_lock_irqsave(&cpufreq_driver_lock, flags);
859 			managed_policy->cpus = policy->cpus;
860 			per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
861 			spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
862 
863 			dprintk("CPU already managed, adding link\n");
864 			ret = sysfs_create_link(&sys_dev->kobj,
865 						&managed_policy->kobj,
866 						"cpufreq");
867 			if (ret)
868 				goto err_out_driver_exit;
869 
870 			cpufreq_debug_enable_ratelimit();
871 			ret = 0;
872 			goto err_out_driver_exit; /* call driver->exit() */
873 		}
874 	}
875 #endif
876 	memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
877 
878 	/* prepare interface data */
879 	ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, &sys_dev->kobj,
880 				   "cpufreq");
881 	if (ret)
882 		goto err_out_driver_exit;
883 
884 	/* set up files for this cpu device */
885 	drv_attr = cpufreq_driver->attr;
886 	while ((drv_attr) && (*drv_attr)) {
887 		ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
888 		if (ret)
889 			goto err_out_driver_exit;
890 		drv_attr++;
891 	}
892 	if (cpufreq_driver->get) {
893 		ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
894 		if (ret)
895 			goto err_out_driver_exit;
896 	}
897 	if (cpufreq_driver->target) {
898 		ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
899 		if (ret)
900 			goto err_out_driver_exit;
901 	}
902 
903 	spin_lock_irqsave(&cpufreq_driver_lock, flags);
904 	for_each_cpu_mask_nr(j, policy->cpus) {
905 		per_cpu(cpufreq_cpu_data, j) = policy;
906 		per_cpu(policy_cpu, j) = policy->cpu;
907 	}
908 	spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
909 
910 	/* symlink affected CPUs */
911 	for_each_cpu_mask_nr(j, policy->cpus) {
912 		if (j == cpu)
913 			continue;
914 		if (!cpu_online(j))
915 			continue;
916 
917 		dprintk("CPU %u already managed, adding link\n", j);
918 		cpufreq_cpu_get(cpu);
919 		cpu_sys_dev = get_cpu_sysdev(j);
920 		ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
921 					"cpufreq");
922 		if (ret)
923 			goto err_out_unregister;
924 	}
925 
926 	policy->governor = NULL; /* to assure that the starting sequence is
927 				  * run in cpufreq_set_policy */
928 
929 	/* set default policy */
930 	ret = __cpufreq_set_policy(policy, &new_policy);
931 	policy->user_policy.policy = policy->policy;
932 	policy->user_policy.governor = policy->governor;
933 
934 	if (ret) {
935 		dprintk("setting policy failed\n");
936 		goto err_out_unregister;
937 	}
938 
939 	unlock_policy_rwsem_write(cpu);
940 
941 	kobject_uevent(&policy->kobj, KOBJ_ADD);
942 	module_put(cpufreq_driver->owner);
943 	dprintk("initialization complete\n");
944 	cpufreq_debug_enable_ratelimit();
945 
946 	return 0;
947 
948 
949 err_out_unregister:
950 	spin_lock_irqsave(&cpufreq_driver_lock, flags);
951 	for_each_cpu_mask_nr(j, policy->cpus)
952 		per_cpu(cpufreq_cpu_data, j) = NULL;
953 	spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
954 
955 	kobject_put(&policy->kobj);
956 	wait_for_completion(&policy->kobj_unregister);
957 
958 err_out_driver_exit:
959 	if (cpufreq_driver->exit)
960 		cpufreq_driver->exit(policy);
961 
962 err_out:
963 	unlock_policy_rwsem_write(cpu);
964 	kfree(policy);
965 
966 nomem_out:
967 	module_put(cpufreq_driver->owner);
968 module_out:
969 	cpufreq_debug_enable_ratelimit();
970 	return ret;
971 }
972 
973 
974 /**
975  * __cpufreq_remove_dev - remove a CPU device
976  *
977  * Removes the cpufreq interface for a CPU device.
978  * Caller should already have policy_rwsem in write mode for this CPU.
979  * This routine frees the rwsem before returning.
980  */
981 static int __cpufreq_remove_dev(struct sys_device *sys_dev)
982 {
983 	unsigned int cpu = sys_dev->id;
984 	unsigned long flags;
985 	struct cpufreq_policy *data;
986 #ifdef CONFIG_SMP
987 	struct sys_device *cpu_sys_dev;
988 	unsigned int j;
989 #endif
990 
991 	cpufreq_debug_disable_ratelimit();
992 	dprintk("unregistering CPU %u\n", cpu);
993 
994 	spin_lock_irqsave(&cpufreq_driver_lock, flags);
995 	data = per_cpu(cpufreq_cpu_data, cpu);
996 
997 	if (!data) {
998 		spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
999 		cpufreq_debug_enable_ratelimit();
1000 		unlock_policy_rwsem_write(cpu);
1001 		return -EINVAL;
1002 	}
1003 	per_cpu(cpufreq_cpu_data, cpu) = NULL;
1004 
1005 
1006 #ifdef CONFIG_SMP
1007 	/* if this isn't the CPU which is the parent of the kobj, we
1008 	 * only need to unlink, put and exit
1009 	 */
1010 	if (unlikely(cpu != data->cpu)) {
1011 		dprintk("removing link\n");
1012 		cpu_clear(cpu, data->cpus);
1013 		spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1014 		sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1015 		cpufreq_cpu_put(data);
1016 		cpufreq_debug_enable_ratelimit();
1017 		unlock_policy_rwsem_write(cpu);
1018 		return 0;
1019 	}
1020 #endif
1021 
1022 #ifdef CONFIG_SMP
1023 
1024 #ifdef CONFIG_HOTPLUG_CPU
1025 	per_cpu(cpufreq_cpu_governor, cpu) = data->governor;
1026 #endif
1027 
1028 	/* if we have other CPUs still registered, we need to unlink them,
1029 	 * or else wait_for_completion below will lock up. Clean the
1030 	 * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1031 	 * the sysfs links afterwards.
1032 	 */
1033 	if (unlikely(cpus_weight(data->cpus) > 1)) {
1034 		for_each_cpu_mask_nr(j, data->cpus) {
1035 			if (j == cpu)
1036 				continue;
1037 			per_cpu(cpufreq_cpu_data, j) = NULL;
1038 		}
1039 	}
1040 
1041 	spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1042 
1043 	if (unlikely(cpus_weight(data->cpus) > 1)) {
1044 		for_each_cpu_mask_nr(j, data->cpus) {
1045 			if (j == cpu)
1046 				continue;
1047 			dprintk("removing link for cpu %u\n", j);
1048 #ifdef CONFIG_HOTPLUG_CPU
1049 			per_cpu(cpufreq_cpu_governor, j) = data->governor;
1050 #endif
1051 			cpu_sys_dev = get_cpu_sysdev(j);
1052 			sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1053 			cpufreq_cpu_put(data);
1054 		}
1055 	}
1056 #else
1057 	spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1058 #endif
1059 
1060 	if (cpufreq_driver->target)
1061 		__cpufreq_governor(data, CPUFREQ_GOV_STOP);
1062 
1063 	unlock_policy_rwsem_write(cpu);
1064 
1065 	kobject_put(&data->kobj);
1066 
1067 	/* we need to make sure that the underlying kobj is actually
1068 	 * not referenced anymore by anybody before we proceed with
1069 	 * unloading.
1070 	 */
1071 	dprintk("waiting for dropping of refcount\n");
1072 	wait_for_completion(&data->kobj_unregister);
1073 	dprintk("wait complete\n");
1074 
1075 	if (cpufreq_driver->exit)
1076 		cpufreq_driver->exit(data);
1077 
1078 	kfree(data);
1079 
1080 	cpufreq_debug_enable_ratelimit();
1081 	return 0;
1082 }
1083 
1084 
1085 static int cpufreq_remove_dev(struct sys_device *sys_dev)
1086 {
1087 	unsigned int cpu = sys_dev->id;
1088 	int retval;
1089 
1090 	if (cpu_is_offline(cpu))
1091 		return 0;
1092 
1093 	if (unlikely(lock_policy_rwsem_write(cpu)))
1094 		BUG();
1095 
1096 	retval = __cpufreq_remove_dev(sys_dev);
1097 	return retval;
1098 }
1099 
1100 
1101 static void handle_update(struct work_struct *work)
1102 {
1103 	struct cpufreq_policy *policy =
1104 		container_of(work, struct cpufreq_policy, update);
1105 	unsigned int cpu = policy->cpu;
1106 	dprintk("handle_update for cpu %u called\n", cpu);
1107 	cpufreq_update_policy(cpu);
1108 }
1109 
1110 /**
1111  *	cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1112  *	@cpu: cpu number
1113  *	@old_freq: CPU frequency the kernel thinks the CPU runs at
1114  *	@new_freq: CPU frequency the CPU actually runs at
1115  *
1116  *	We adjust to current frequency first, and need to clean up later. So either call
1117  *	to cpufreq_update_policy() or schedule handle_update()).
1118  */
1119 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1120 				unsigned int new_freq)
1121 {
1122 	struct cpufreq_freqs freqs;
1123 
1124 	dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1125 	       "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1126 
1127 	freqs.cpu = cpu;
1128 	freqs.old = old_freq;
1129 	freqs.new = new_freq;
1130 	cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1131 	cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1132 }
1133 
1134 
1135 /**
1136  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1137  * @cpu: CPU number
1138  *
1139  * This is the last known freq, without actually getting it from the driver.
1140  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1141  */
1142 unsigned int cpufreq_quick_get(unsigned int cpu)
1143 {
1144 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1145 	unsigned int ret_freq = 0;
1146 
1147 	if (policy) {
1148 		ret_freq = policy->cur;
1149 		cpufreq_cpu_put(policy);
1150 	}
1151 
1152 	return ret_freq;
1153 }
1154 EXPORT_SYMBOL(cpufreq_quick_get);
1155 
1156 
1157 static unsigned int __cpufreq_get(unsigned int cpu)
1158 {
1159 	struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1160 	unsigned int ret_freq = 0;
1161 
1162 	if (!cpufreq_driver->get)
1163 		return ret_freq;
1164 
1165 	ret_freq = cpufreq_driver->get(cpu);
1166 
1167 	if (ret_freq && policy->cur &&
1168 		!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1169 		/* verify no discrepancy between actual and
1170 					saved value exists */
1171 		if (unlikely(ret_freq != policy->cur)) {
1172 			cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1173 			schedule_work(&policy->update);
1174 		}
1175 	}
1176 
1177 	return ret_freq;
1178 }
1179 
1180 /**
1181  * cpufreq_get - get the current CPU frequency (in kHz)
1182  * @cpu: CPU number
1183  *
1184  * Get the CPU current (static) CPU frequency
1185  */
1186 unsigned int cpufreq_get(unsigned int cpu)
1187 {
1188 	unsigned int ret_freq = 0;
1189 	struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1190 
1191 	if (!policy)
1192 		goto out;
1193 
1194 	if (unlikely(lock_policy_rwsem_read(cpu)))
1195 		goto out_policy;
1196 
1197 	ret_freq = __cpufreq_get(cpu);
1198 
1199 	unlock_policy_rwsem_read(cpu);
1200 
1201 out_policy:
1202 	cpufreq_cpu_put(policy);
1203 out:
1204 	return ret_freq;
1205 }
1206 EXPORT_SYMBOL(cpufreq_get);
1207 
1208 
1209 /**
1210  *	cpufreq_suspend - let the low level driver prepare for suspend
1211  */
1212 
1213 static int cpufreq_suspend(struct sys_device *sysdev, pm_message_t pmsg)
1214 {
1215 	int cpu = sysdev->id;
1216 	int ret = 0;
1217 	unsigned int cur_freq = 0;
1218 	struct cpufreq_policy *cpu_policy;
1219 
1220 	dprintk("suspending cpu %u\n", cpu);
1221 
1222 	if (!cpu_online(cpu))
1223 		return 0;
1224 
1225 	/* we may be lax here as interrupts are off. Nonetheless
1226 	 * we need to grab the correct cpu policy, as to check
1227 	 * whether we really run on this CPU.
1228 	 */
1229 
1230 	cpu_policy = cpufreq_cpu_get(cpu);
1231 	if (!cpu_policy)
1232 		return -EINVAL;
1233 
1234 	/* only handle each CPU group once */
1235 	if (unlikely(cpu_policy->cpu != cpu))
1236 		goto out;
1237 
1238 	if (cpufreq_driver->suspend) {
1239 		ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1240 		if (ret) {
1241 			printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1242 					"step on CPU %u\n", cpu_policy->cpu);
1243 			goto out;
1244 		}
1245 	}
1246 
1247 	if (cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)
1248 		goto out;
1249 
1250 	if (cpufreq_driver->get)
1251 		cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1252 
1253 	if (!cur_freq || !cpu_policy->cur) {
1254 		printk(KERN_ERR "cpufreq: suspend failed to assert current "
1255 		       "frequency is what timing core thinks it is.\n");
1256 		goto out;
1257 	}
1258 
1259 	if (unlikely(cur_freq != cpu_policy->cur)) {
1260 		struct cpufreq_freqs freqs;
1261 
1262 		if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1263 			dprintk("Warning: CPU frequency is %u, "
1264 			       "cpufreq assumed %u kHz.\n",
1265 			       cur_freq, cpu_policy->cur);
1266 
1267 		freqs.cpu = cpu;
1268 		freqs.old = cpu_policy->cur;
1269 		freqs.new = cur_freq;
1270 
1271 		srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
1272 				    CPUFREQ_SUSPENDCHANGE, &freqs);
1273 		adjust_jiffies(CPUFREQ_SUSPENDCHANGE, &freqs);
1274 
1275 		cpu_policy->cur = cur_freq;
1276 	}
1277 
1278 out:
1279 	cpufreq_cpu_put(cpu_policy);
1280 	return ret;
1281 }
1282 
1283 /**
1284  *	cpufreq_resume -  restore proper CPU frequency handling after resume
1285  *
1286  *	1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1287  *	2.) if ->target and !CPUFREQ_CONST_LOOPS: verify we're in sync
1288  *	3.) schedule call cpufreq_update_policy() ASAP as interrupts are
1289  *	    restored.
1290  */
1291 static int cpufreq_resume(struct sys_device *sysdev)
1292 {
1293 	int cpu = sysdev->id;
1294 	int ret = 0;
1295 	struct cpufreq_policy *cpu_policy;
1296 
1297 	dprintk("resuming cpu %u\n", cpu);
1298 
1299 	if (!cpu_online(cpu))
1300 		return 0;
1301 
1302 	/* we may be lax here as interrupts are off. Nonetheless
1303 	 * we need to grab the correct cpu policy, as to check
1304 	 * whether we really run on this CPU.
1305 	 */
1306 
1307 	cpu_policy = cpufreq_cpu_get(cpu);
1308 	if (!cpu_policy)
1309 		return -EINVAL;
1310 
1311 	/* only handle each CPU group once */
1312 	if (unlikely(cpu_policy->cpu != cpu))
1313 		goto fail;
1314 
1315 	if (cpufreq_driver->resume) {
1316 		ret = cpufreq_driver->resume(cpu_policy);
1317 		if (ret) {
1318 			printk(KERN_ERR "cpufreq: resume failed in ->resume "
1319 					"step on CPU %u\n", cpu_policy->cpu);
1320 			goto fail;
1321 		}
1322 	}
1323 
1324 	if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1325 		unsigned int cur_freq = 0;
1326 
1327 		if (cpufreq_driver->get)
1328 			cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1329 
1330 		if (!cur_freq || !cpu_policy->cur) {
1331 			printk(KERN_ERR "cpufreq: resume failed to assert "
1332 					"current frequency is what timing core "
1333 					"thinks it is.\n");
1334 			goto out;
1335 		}
1336 
1337 		if (unlikely(cur_freq != cpu_policy->cur)) {
1338 			struct cpufreq_freqs freqs;
1339 
1340 			if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1341 				dprintk("Warning: CPU frequency "
1342 				       "is %u, cpufreq assumed %u kHz.\n",
1343 				       cur_freq, cpu_policy->cur);
1344 
1345 			freqs.cpu = cpu;
1346 			freqs.old = cpu_policy->cur;
1347 			freqs.new = cur_freq;
1348 
1349 			srcu_notifier_call_chain(
1350 					&cpufreq_transition_notifier_list,
1351 					CPUFREQ_RESUMECHANGE, &freqs);
1352 			adjust_jiffies(CPUFREQ_RESUMECHANGE, &freqs);
1353 
1354 			cpu_policy->cur = cur_freq;
1355 		}
1356 	}
1357 
1358 out:
1359 	schedule_work(&cpu_policy->update);
1360 fail:
1361 	cpufreq_cpu_put(cpu_policy);
1362 	return ret;
1363 }
1364 
1365 static struct sysdev_driver cpufreq_sysdev_driver = {
1366 	.add		= cpufreq_add_dev,
1367 	.remove		= cpufreq_remove_dev,
1368 	.suspend	= cpufreq_suspend,
1369 	.resume		= cpufreq_resume,
1370 };
1371 
1372 
1373 /*********************************************************************
1374  *                     NOTIFIER LISTS INTERFACE                      *
1375  *********************************************************************/
1376 
1377 /**
1378  *	cpufreq_register_notifier - register a driver with cpufreq
1379  *	@nb: notifier function to register
1380  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1381  *
1382  *	Add a driver to one of two lists: either a list of drivers that
1383  *      are notified about clock rate changes (once before and once after
1384  *      the transition), or a list of drivers that are notified about
1385  *      changes in cpufreq policy.
1386  *
1387  *	This function may sleep, and has the same return conditions as
1388  *	blocking_notifier_chain_register.
1389  */
1390 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1391 {
1392 	int ret;
1393 
1394 	WARN_ON(!init_cpufreq_transition_notifier_list_called);
1395 
1396 	switch (list) {
1397 	case CPUFREQ_TRANSITION_NOTIFIER:
1398 		ret = srcu_notifier_chain_register(
1399 				&cpufreq_transition_notifier_list, nb);
1400 		break;
1401 	case CPUFREQ_POLICY_NOTIFIER:
1402 		ret = blocking_notifier_chain_register(
1403 				&cpufreq_policy_notifier_list, nb);
1404 		break;
1405 	default:
1406 		ret = -EINVAL;
1407 	}
1408 
1409 	return ret;
1410 }
1411 EXPORT_SYMBOL(cpufreq_register_notifier);
1412 
1413 
1414 /**
1415  *	cpufreq_unregister_notifier - unregister a driver with cpufreq
1416  *	@nb: notifier block to be unregistered
1417  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1418  *
1419  *	Remove a driver from the CPU frequency notifier list.
1420  *
1421  *	This function may sleep, and has the same return conditions as
1422  *	blocking_notifier_chain_unregister.
1423  */
1424 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1425 {
1426 	int ret;
1427 
1428 	switch (list) {
1429 	case CPUFREQ_TRANSITION_NOTIFIER:
1430 		ret = srcu_notifier_chain_unregister(
1431 				&cpufreq_transition_notifier_list, nb);
1432 		break;
1433 	case CPUFREQ_POLICY_NOTIFIER:
1434 		ret = blocking_notifier_chain_unregister(
1435 				&cpufreq_policy_notifier_list, nb);
1436 		break;
1437 	default:
1438 		ret = -EINVAL;
1439 	}
1440 
1441 	return ret;
1442 }
1443 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1444 
1445 
1446 /*********************************************************************
1447  *                              GOVERNORS                            *
1448  *********************************************************************/
1449 
1450 
1451 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1452 			    unsigned int target_freq,
1453 			    unsigned int relation)
1454 {
1455 	int retval = -EINVAL;
1456 
1457 	dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1458 		target_freq, relation);
1459 	if (cpu_online(policy->cpu) && cpufreq_driver->target)
1460 		retval = cpufreq_driver->target(policy, target_freq, relation);
1461 
1462 	return retval;
1463 }
1464 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1465 
1466 int cpufreq_driver_target(struct cpufreq_policy *policy,
1467 			  unsigned int target_freq,
1468 			  unsigned int relation)
1469 {
1470 	int ret = -EINVAL;
1471 
1472 	policy = cpufreq_cpu_get(policy->cpu);
1473 	if (!policy)
1474 		goto no_policy;
1475 
1476 	if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1477 		goto fail;
1478 
1479 	ret = __cpufreq_driver_target(policy, target_freq, relation);
1480 
1481 	unlock_policy_rwsem_write(policy->cpu);
1482 
1483 fail:
1484 	cpufreq_cpu_put(policy);
1485 no_policy:
1486 	return ret;
1487 }
1488 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1489 
1490 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1491 {
1492 	int ret = 0;
1493 
1494 	policy = cpufreq_cpu_get(policy->cpu);
1495 	if (!policy)
1496 		return -EINVAL;
1497 
1498 	if (cpu_online(cpu) && cpufreq_driver->getavg)
1499 		ret = cpufreq_driver->getavg(policy, cpu);
1500 
1501 	cpufreq_cpu_put(policy);
1502 	return ret;
1503 }
1504 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1505 
1506 /*
1507  * when "event" is CPUFREQ_GOV_LIMITS
1508  */
1509 
1510 static int __cpufreq_governor(struct cpufreq_policy *policy,
1511 					unsigned int event)
1512 {
1513 	int ret;
1514 
1515 	/* Only must be defined when default governor is known to have latency
1516 	   restrictions, like e.g. conservative or ondemand.
1517 	   That this is the case is already ensured in Kconfig
1518 	*/
1519 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1520 	struct cpufreq_governor *gov = &cpufreq_gov_performance;
1521 #else
1522 	struct cpufreq_governor *gov = NULL;
1523 #endif
1524 
1525 	if (policy->governor->max_transition_latency &&
1526 	    policy->cpuinfo.transition_latency >
1527 	    policy->governor->max_transition_latency) {
1528 		if (!gov)
1529 			return -EINVAL;
1530 		else {
1531 			printk(KERN_WARNING "%s governor failed, too long"
1532 			       " transition latency of HW, fallback"
1533 			       " to %s governor\n",
1534 			       policy->governor->name,
1535 			       gov->name);
1536 			policy->governor = gov;
1537 		}
1538 	}
1539 
1540 	if (!try_module_get(policy->governor->owner))
1541 		return -EINVAL;
1542 
1543 	dprintk("__cpufreq_governor for CPU %u, event %u\n",
1544 						policy->cpu, event);
1545 	ret = policy->governor->governor(policy, event);
1546 
1547 	/* we keep one module reference alive for
1548 			each CPU governed by this CPU */
1549 	if ((event != CPUFREQ_GOV_START) || ret)
1550 		module_put(policy->governor->owner);
1551 	if ((event == CPUFREQ_GOV_STOP) && !ret)
1552 		module_put(policy->governor->owner);
1553 
1554 	return ret;
1555 }
1556 
1557 
1558 int cpufreq_register_governor(struct cpufreq_governor *governor)
1559 {
1560 	int err;
1561 
1562 	if (!governor)
1563 		return -EINVAL;
1564 
1565 	mutex_lock(&cpufreq_governor_mutex);
1566 
1567 	err = -EBUSY;
1568 	if (__find_governor(governor->name) == NULL) {
1569 		err = 0;
1570 		list_add(&governor->governor_list, &cpufreq_governor_list);
1571 	}
1572 
1573 	mutex_unlock(&cpufreq_governor_mutex);
1574 	return err;
1575 }
1576 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1577 
1578 
1579 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1580 {
1581 	if (!governor)
1582 		return;
1583 
1584 	mutex_lock(&cpufreq_governor_mutex);
1585 	list_del(&governor->governor_list);
1586 	mutex_unlock(&cpufreq_governor_mutex);
1587 	return;
1588 }
1589 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1590 
1591 
1592 
1593 /*********************************************************************
1594  *                          POLICY INTERFACE                         *
1595  *********************************************************************/
1596 
1597 /**
1598  * cpufreq_get_policy - get the current cpufreq_policy
1599  * @policy: struct cpufreq_policy into which the current cpufreq_policy is written
1600  *
1601  * Reads the current cpufreq policy.
1602  */
1603 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1604 {
1605 	struct cpufreq_policy *cpu_policy;
1606 	if (!policy)
1607 		return -EINVAL;
1608 
1609 	cpu_policy = cpufreq_cpu_get(cpu);
1610 	if (!cpu_policy)
1611 		return -EINVAL;
1612 
1613 	memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1614 
1615 	cpufreq_cpu_put(cpu_policy);
1616 	return 0;
1617 }
1618 EXPORT_SYMBOL(cpufreq_get_policy);
1619 
1620 
1621 /*
1622  * data   : current policy.
1623  * policy : policy to be set.
1624  */
1625 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1626 				struct cpufreq_policy *policy)
1627 {
1628 	int ret = 0;
1629 
1630 	cpufreq_debug_disable_ratelimit();
1631 	dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1632 		policy->min, policy->max);
1633 
1634 	memcpy(&policy->cpuinfo, &data->cpuinfo,
1635 				sizeof(struct cpufreq_cpuinfo));
1636 
1637 	if (policy->min > data->max || policy->max < data->min) {
1638 		ret = -EINVAL;
1639 		goto error_out;
1640 	}
1641 
1642 	/* verify the cpu speed can be set within this limit */
1643 	ret = cpufreq_driver->verify(policy);
1644 	if (ret)
1645 		goto error_out;
1646 
1647 	/* adjust if necessary - all reasons */
1648 	blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1649 			CPUFREQ_ADJUST, policy);
1650 
1651 	/* adjust if necessary - hardware incompatibility*/
1652 	blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1653 			CPUFREQ_INCOMPATIBLE, policy);
1654 
1655 	/* verify the cpu speed can be set within this limit,
1656 	   which might be different to the first one */
1657 	ret = cpufreq_driver->verify(policy);
1658 	if (ret)
1659 		goto error_out;
1660 
1661 	/* notification of the new policy */
1662 	blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1663 			CPUFREQ_NOTIFY, policy);
1664 
1665 	data->min = policy->min;
1666 	data->max = policy->max;
1667 
1668 	dprintk("new min and max freqs are %u - %u kHz\n",
1669 					data->min, data->max);
1670 
1671 	if (cpufreq_driver->setpolicy) {
1672 		data->policy = policy->policy;
1673 		dprintk("setting range\n");
1674 		ret = cpufreq_driver->setpolicy(policy);
1675 	} else {
1676 		if (policy->governor != data->governor) {
1677 			/* save old, working values */
1678 			struct cpufreq_governor *old_gov = data->governor;
1679 
1680 			dprintk("governor switch\n");
1681 
1682 			/* end old governor */
1683 			if (data->governor)
1684 				__cpufreq_governor(data, CPUFREQ_GOV_STOP);
1685 
1686 			/* start new governor */
1687 			data->governor = policy->governor;
1688 			if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1689 				/* new governor failed, so re-start old one */
1690 				dprintk("starting governor %s failed\n",
1691 							data->governor->name);
1692 				if (old_gov) {
1693 					data->governor = old_gov;
1694 					__cpufreq_governor(data,
1695 							   CPUFREQ_GOV_START);
1696 				}
1697 				ret = -EINVAL;
1698 				goto error_out;
1699 			}
1700 			/* might be a policy change, too, so fall through */
1701 		}
1702 		dprintk("governor: change or update limits\n");
1703 		__cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1704 	}
1705 
1706 error_out:
1707 	cpufreq_debug_enable_ratelimit();
1708 	return ret;
1709 }
1710 
1711 /**
1712  *	cpufreq_update_policy - re-evaluate an existing cpufreq policy
1713  *	@cpu: CPU which shall be re-evaluated
1714  *
1715  *	Usefull for policy notifiers which have different necessities
1716  *	at different times.
1717  */
1718 int cpufreq_update_policy(unsigned int cpu)
1719 {
1720 	struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1721 	struct cpufreq_policy policy;
1722 	int ret;
1723 
1724 	if (!data) {
1725 		ret = -ENODEV;
1726 		goto no_policy;
1727 	}
1728 
1729 	if (unlikely(lock_policy_rwsem_write(cpu))) {
1730 		ret = -EINVAL;
1731 		goto fail;
1732 	}
1733 
1734 	dprintk("updating policy for CPU %u\n", cpu);
1735 	memcpy(&policy, data, sizeof(struct cpufreq_policy));
1736 	policy.min = data->user_policy.min;
1737 	policy.max = data->user_policy.max;
1738 	policy.policy = data->user_policy.policy;
1739 	policy.governor = data->user_policy.governor;
1740 
1741 	/* BIOS might change freq behind our back
1742 	  -> ask driver for current freq and notify governors about a change */
1743 	if (cpufreq_driver->get) {
1744 		policy.cur = cpufreq_driver->get(cpu);
1745 		if (!data->cur) {
1746 			dprintk("Driver did not initialize current freq");
1747 			data->cur = policy.cur;
1748 		} else {
1749 			if (data->cur != policy.cur)
1750 				cpufreq_out_of_sync(cpu, data->cur,
1751 								policy.cur);
1752 		}
1753 	}
1754 
1755 	ret = __cpufreq_set_policy(data, &policy);
1756 
1757 	unlock_policy_rwsem_write(cpu);
1758 
1759 fail:
1760 	cpufreq_cpu_put(data);
1761 no_policy:
1762 	return ret;
1763 }
1764 EXPORT_SYMBOL(cpufreq_update_policy);
1765 
1766 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1767 					unsigned long action, void *hcpu)
1768 {
1769 	unsigned int cpu = (unsigned long)hcpu;
1770 	struct sys_device *sys_dev;
1771 
1772 	sys_dev = get_cpu_sysdev(cpu);
1773 	if (sys_dev) {
1774 		switch (action) {
1775 		case CPU_ONLINE:
1776 		case CPU_ONLINE_FROZEN:
1777 			cpufreq_add_dev(sys_dev);
1778 			break;
1779 		case CPU_DOWN_PREPARE:
1780 		case CPU_DOWN_PREPARE_FROZEN:
1781 			if (unlikely(lock_policy_rwsem_write(cpu)))
1782 				BUG();
1783 
1784 			__cpufreq_remove_dev(sys_dev);
1785 			break;
1786 		case CPU_DOWN_FAILED:
1787 		case CPU_DOWN_FAILED_FROZEN:
1788 			cpufreq_add_dev(sys_dev);
1789 			break;
1790 		}
1791 	}
1792 	return NOTIFY_OK;
1793 }
1794 
1795 static struct notifier_block __refdata cpufreq_cpu_notifier =
1796 {
1797     .notifier_call = cpufreq_cpu_callback,
1798 };
1799 
1800 /*********************************************************************
1801  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1802  *********************************************************************/
1803 
1804 /**
1805  * cpufreq_register_driver - register a CPU Frequency driver
1806  * @driver_data: A struct cpufreq_driver containing the values#
1807  * submitted by the CPU Frequency driver.
1808  *
1809  *   Registers a CPU Frequency driver to this core code. This code
1810  * returns zero on success, -EBUSY when another driver got here first
1811  * (and isn't unregistered in the meantime).
1812  *
1813  */
1814 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1815 {
1816 	unsigned long flags;
1817 	int ret;
1818 
1819 	if (!driver_data || !driver_data->verify || !driver_data->init ||
1820 	    ((!driver_data->setpolicy) && (!driver_data->target)))
1821 		return -EINVAL;
1822 
1823 	dprintk("trying to register driver %s\n", driver_data->name);
1824 
1825 	if (driver_data->setpolicy)
1826 		driver_data->flags |= CPUFREQ_CONST_LOOPS;
1827 
1828 	spin_lock_irqsave(&cpufreq_driver_lock, flags);
1829 	if (cpufreq_driver) {
1830 		spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1831 		return -EBUSY;
1832 	}
1833 	cpufreq_driver = driver_data;
1834 	spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1835 
1836 	ret = sysdev_driver_register(&cpu_sysdev_class,
1837 					&cpufreq_sysdev_driver);
1838 
1839 	if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1840 		int i;
1841 		ret = -ENODEV;
1842 
1843 		/* check for at least one working CPU */
1844 		for (i = 0; i < nr_cpu_ids; i++)
1845 			if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1846 				ret = 0;
1847 				break;
1848 			}
1849 
1850 		/* if all ->init() calls failed, unregister */
1851 		if (ret) {
1852 			dprintk("no CPU initialized for driver %s\n",
1853 							driver_data->name);
1854 			sysdev_driver_unregister(&cpu_sysdev_class,
1855 						&cpufreq_sysdev_driver);
1856 
1857 			spin_lock_irqsave(&cpufreq_driver_lock, flags);
1858 			cpufreq_driver = NULL;
1859 			spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1860 		}
1861 	}
1862 
1863 	if (!ret) {
1864 		register_hotcpu_notifier(&cpufreq_cpu_notifier);
1865 		dprintk("driver %s up and running\n", driver_data->name);
1866 		cpufreq_debug_enable_ratelimit();
1867 	}
1868 
1869 	return ret;
1870 }
1871 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1872 
1873 
1874 /**
1875  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1876  *
1877  *    Unregister the current CPUFreq driver. Only call this if you have
1878  * the right to do so, i.e. if you have succeeded in initialising before!
1879  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1880  * currently not initialised.
1881  */
1882 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1883 {
1884 	unsigned long flags;
1885 
1886 	cpufreq_debug_disable_ratelimit();
1887 
1888 	if (!cpufreq_driver || (driver != cpufreq_driver)) {
1889 		cpufreq_debug_enable_ratelimit();
1890 		return -EINVAL;
1891 	}
1892 
1893 	dprintk("unregistering driver %s\n", driver->name);
1894 
1895 	sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1896 	unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1897 
1898 	spin_lock_irqsave(&cpufreq_driver_lock, flags);
1899 	cpufreq_driver = NULL;
1900 	spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1901 
1902 	return 0;
1903 }
1904 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1905 
1906 static int __init cpufreq_core_init(void)
1907 {
1908 	int cpu;
1909 
1910 	for_each_possible_cpu(cpu) {
1911 		per_cpu(policy_cpu, cpu) = -1;
1912 		init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1913 	}
1914 	return 0;
1915 }
1916 
1917 core_initcall(cpufreq_core_init);
1918