xref: /openbmc/linux/drivers/cpufreq/intel_pstate.c (revision c0d6fe2f01c475cc137d90607a07578586883df8)
1 /*
2  * intel_pstate.c: Native P state management for Intel processors
3  *
4  * (C) Copyright 2012 Intel Corporation
5  * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
6  *
7  * This program is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU General Public License
9  * as published by the Free Software Foundation; version 2
10  * of the License.
11  */
12 
13 #include <linux/kernel.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/module.h>
16 #include <linux/ktime.h>
17 #include <linux/hrtimer.h>
18 #include <linux/tick.h>
19 #include <linux/slab.h>
20 #include <linux/sched.h>
21 #include <linux/list.h>
22 #include <linux/cpu.h>
23 #include <linux/cpufreq.h>
24 #include <linux/sysfs.h>
25 #include <linux/types.h>
26 #include <linux/fs.h>
27 #include <linux/debugfs.h>
28 #include <linux/acpi.h>
29 #include <linux/vmalloc.h>
30 #include <trace/events/power.h>
31 
32 #include <asm/div64.h>
33 #include <asm/msr.h>
34 #include <asm/cpu_device_id.h>
35 #include <asm/cpufeature.h>
36 
37 #if IS_ENABLED(CONFIG_ACPI)
38 #include <acpi/processor.h>
39 #endif
40 
41 #define BYT_RATIOS		0x66a
42 #define BYT_VIDS		0x66b
43 #define BYT_TURBO_RATIOS	0x66c
44 #define BYT_TURBO_VIDS		0x66d
45 
46 #define FRAC_BITS 8
47 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
48 #define fp_toint(X) ((X) >> FRAC_BITS)
49 
50 static inline int32_t mul_fp(int32_t x, int32_t y)
51 {
52 	return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
53 }
54 
55 static inline int32_t div_fp(s64 x, s64 y)
56 {
57 	return div64_s64((int64_t)x << FRAC_BITS, y);
58 }
59 
60 static inline int ceiling_fp(int32_t x)
61 {
62 	int mask, ret;
63 
64 	ret = fp_toint(x);
65 	mask = (1 << FRAC_BITS) - 1;
66 	if (x & mask)
67 		ret += 1;
68 	return ret;
69 }
70 
71 struct sample {
72 	int32_t core_pct_busy;
73 	u64 aperf;
74 	u64 mperf;
75 	u64 tsc;
76 	int freq;
77 	ktime_t time;
78 };
79 
80 struct pstate_data {
81 	int	current_pstate;
82 	int	min_pstate;
83 	int	max_pstate;
84 	int	max_pstate_physical;
85 	int	scaling;
86 	int	turbo_pstate;
87 };
88 
89 struct vid_data {
90 	int min;
91 	int max;
92 	int turbo;
93 	int32_t ratio;
94 };
95 
96 struct _pid {
97 	int setpoint;
98 	int32_t integral;
99 	int32_t p_gain;
100 	int32_t i_gain;
101 	int32_t d_gain;
102 	int deadband;
103 	int32_t last_err;
104 };
105 
106 struct cpudata {
107 	int cpu;
108 
109 	struct timer_list timer;
110 
111 	struct pstate_data pstate;
112 	struct vid_data vid;
113 	struct _pid pid;
114 
115 	ktime_t last_sample_time;
116 	u64	prev_aperf;
117 	u64	prev_mperf;
118 	u64	prev_tsc;
119 	struct sample sample;
120 #if IS_ENABLED(CONFIG_ACPI)
121 	struct acpi_processor_performance acpi_perf_data;
122 #endif
123 };
124 
125 static struct cpudata **all_cpu_data;
126 struct pstate_adjust_policy {
127 	int sample_rate_ms;
128 	int deadband;
129 	int setpoint;
130 	int p_gain_pct;
131 	int d_gain_pct;
132 	int i_gain_pct;
133 };
134 
135 struct pstate_funcs {
136 	int (*get_max)(void);
137 	int (*get_max_physical)(void);
138 	int (*get_min)(void);
139 	int (*get_turbo)(void);
140 	int (*get_scaling)(void);
141 	void (*set)(struct cpudata*, int pstate);
142 	void (*get_vid)(struct cpudata *);
143 };
144 
145 struct cpu_defaults {
146 	struct pstate_adjust_policy pid_policy;
147 	struct pstate_funcs funcs;
148 };
149 
150 static struct pstate_adjust_policy pid_params;
151 static struct pstate_funcs pstate_funcs;
152 static int hwp_active;
153 static int no_acpi_perf;
154 
155 struct perf_limits {
156 	int no_turbo;
157 	int turbo_disabled;
158 	int max_perf_pct;
159 	int min_perf_pct;
160 	int32_t max_perf;
161 	int32_t min_perf;
162 	int max_policy_pct;
163 	int max_sysfs_pct;
164 	int min_policy_pct;
165 	int min_sysfs_pct;
166 	int max_perf_ctl;
167 	int min_perf_ctl;
168 };
169 
170 static struct perf_limits performance_limits = {
171 	.no_turbo = 0,
172 	.turbo_disabled = 0,
173 	.max_perf_pct = 100,
174 	.max_perf = int_tofp(1),
175 	.min_perf_pct = 100,
176 	.min_perf = int_tofp(1),
177 	.max_policy_pct = 100,
178 	.max_sysfs_pct = 100,
179 	.min_policy_pct = 0,
180 	.min_sysfs_pct = 0,
181 };
182 
183 static struct perf_limits powersave_limits = {
184 	.no_turbo = 0,
185 	.turbo_disabled = 0,
186 	.max_perf_pct = 100,
187 	.max_perf = int_tofp(1),
188 	.min_perf_pct = 0,
189 	.min_perf = 0,
190 	.max_policy_pct = 100,
191 	.max_sysfs_pct = 100,
192 	.min_policy_pct = 0,
193 	.min_sysfs_pct = 0,
194 	.max_perf_ctl = 0,
195 	.min_perf_ctl = 0,
196 };
197 
198 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE
199 static struct perf_limits *limits = &performance_limits;
200 #else
201 static struct perf_limits *limits = &powersave_limits;
202 #endif
203 
204 #if IS_ENABLED(CONFIG_ACPI)
205 /*
206  * The max target pstate ratio is a 8 bit value in both PLATFORM_INFO MSR and
207  * in TURBO_RATIO_LIMIT MSR, which pstate driver stores in max_pstate and
208  * max_turbo_pstate fields. The PERF_CTL MSR contains 16 bit value for P state
209  * ratio, out of it only high 8 bits are used. For example 0x1700 is setting
210  * target ratio 0x17. The _PSS control value stores in a format which can be
211  * directly written to PERF_CTL MSR. But in intel_pstate driver this shift
212  * occurs during write to PERF_CTL (E.g. for cores core_set_pstate()).
213  * This function converts the _PSS control value to intel pstate driver format
214  * for comparison and assignment.
215  */
216 static int convert_to_native_pstate_format(struct cpudata *cpu, int index)
217 {
218 	return cpu->acpi_perf_data.states[index].control >> 8;
219 }
220 
221 static int intel_pstate_init_perf_limits(struct cpufreq_policy *policy)
222 {
223 	struct cpudata *cpu;
224 	int ret;
225 	bool turbo_absent = false;
226 	int max_pstate_index;
227 	int min_pss_ctl, max_pss_ctl, turbo_pss_ctl;
228 	int i;
229 
230 	cpu = all_cpu_data[policy->cpu];
231 
232 	pr_debug("intel_pstate: default limits 0x%x 0x%x 0x%x\n",
233 		 cpu->pstate.min_pstate, cpu->pstate.max_pstate,
234 		 cpu->pstate.turbo_pstate);
235 
236 	if (!cpu->acpi_perf_data.shared_cpu_map &&
237 	    zalloc_cpumask_var_node(&cpu->acpi_perf_data.shared_cpu_map,
238 				    GFP_KERNEL, cpu_to_node(policy->cpu))) {
239 		return -ENOMEM;
240 	}
241 
242 	ret = acpi_processor_register_performance(&cpu->acpi_perf_data,
243 						  policy->cpu);
244 	if (ret)
245 		return ret;
246 
247 	/*
248 	 * Check if the control value in _PSS is for PERF_CTL MSR, which should
249 	 * guarantee that the states returned by it map to the states in our
250 	 * list directly.
251 	 */
252 	if (cpu->acpi_perf_data.control_register.space_id !=
253 						ACPI_ADR_SPACE_FIXED_HARDWARE)
254 		return -EIO;
255 
256 	pr_debug("intel_pstate: CPU%u - ACPI _PSS perf data\n", policy->cpu);
257 	for (i = 0; i < cpu->acpi_perf_data.state_count; i++)
258 		pr_debug("     %cP%d: %u MHz, %u mW, 0x%x\n",
259 			 (i == cpu->acpi_perf_data.state ? '*' : ' '), i,
260 			 (u32) cpu->acpi_perf_data.states[i].core_frequency,
261 			 (u32) cpu->acpi_perf_data.states[i].power,
262 			 (u32) cpu->acpi_perf_data.states[i].control);
263 
264 	/*
265 	 * If there is only one entry _PSS, simply ignore _PSS and continue as
266 	 * usual without taking _PSS into account
267 	 */
268 	if (cpu->acpi_perf_data.state_count < 2)
269 		return 0;
270 
271 	turbo_pss_ctl = convert_to_native_pstate_format(cpu, 0);
272 	min_pss_ctl = convert_to_native_pstate_format(cpu,
273 					cpu->acpi_perf_data.state_count - 1);
274 	/* Check if there is a turbo freq in _PSS */
275 	if (turbo_pss_ctl <= cpu->pstate.max_pstate &&
276 	    turbo_pss_ctl > cpu->pstate.min_pstate) {
277 		pr_debug("intel_pstate: no turbo range exists in _PSS\n");
278 		limits->no_turbo = limits->turbo_disabled = 1;
279 		cpu->pstate.turbo_pstate = cpu->pstate.max_pstate;
280 		turbo_absent = true;
281 	}
282 
283 	/* Check if the max non turbo p state < Intel P state max */
284 	max_pstate_index = turbo_absent ? 0 : 1;
285 	max_pss_ctl = convert_to_native_pstate_format(cpu, max_pstate_index);
286 	if (max_pss_ctl < cpu->pstate.max_pstate &&
287 	    max_pss_ctl > cpu->pstate.min_pstate)
288 		cpu->pstate.max_pstate = max_pss_ctl;
289 
290 	/* check If min perf > Intel P State min */
291 	if (min_pss_ctl > cpu->pstate.min_pstate &&
292 	    min_pss_ctl < cpu->pstate.max_pstate) {
293 		cpu->pstate.min_pstate = min_pss_ctl;
294 		policy->cpuinfo.min_freq = min_pss_ctl * cpu->pstate.scaling;
295 	}
296 
297 	if (turbo_absent)
298 		policy->cpuinfo.max_freq = cpu->pstate.max_pstate *
299 						cpu->pstate.scaling;
300 	else {
301 		policy->cpuinfo.max_freq = cpu->pstate.turbo_pstate *
302 						cpu->pstate.scaling;
303 		/*
304 		 * The _PSS table doesn't contain whole turbo frequency range.
305 		 * This just contains +1 MHZ above the max non turbo frequency,
306 		 * with control value corresponding to max turbo ratio. But
307 		 * when cpufreq set policy is called, it will call with this
308 		 * max frequency, which will cause a reduced performance as
309 		 * this driver uses real max turbo frequency as the max
310 		 * frequeny. So correct this frequency in _PSS table to
311 		 * correct max turbo frequency based on the turbo ratio.
312 		 * Also need to convert to MHz as _PSS freq is in MHz.
313 		 */
314 		cpu->acpi_perf_data.states[0].core_frequency =
315 						turbo_pss_ctl * 100;
316 	}
317 
318 	pr_debug("intel_pstate: Updated limits using _PSS 0x%x 0x%x 0x%x\n",
319 		 cpu->pstate.min_pstate, cpu->pstate.max_pstate,
320 		 cpu->pstate.turbo_pstate);
321 	pr_debug("intel_pstate: policy max_freq=%d Khz min_freq = %d KHz\n",
322 		 policy->cpuinfo.max_freq, policy->cpuinfo.min_freq);
323 
324 	return 0;
325 }
326 
327 static int intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
328 {
329 	struct cpudata *cpu;
330 
331 	if (!no_acpi_perf)
332 		return 0;
333 
334 	cpu = all_cpu_data[policy->cpu];
335 	acpi_processor_unregister_performance(policy->cpu);
336 	return 0;
337 }
338 
339 #else
340 static int intel_pstate_init_perf_limits(struct cpufreq_policy *policy)
341 {
342 	return 0;
343 }
344 
345 static int intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
346 {
347 	return 0;
348 }
349 #endif
350 
351 static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
352 			     int deadband, int integral) {
353 	pid->setpoint = setpoint;
354 	pid->deadband  = deadband;
355 	pid->integral  = int_tofp(integral);
356 	pid->last_err  = int_tofp(setpoint) - int_tofp(busy);
357 }
358 
359 static inline void pid_p_gain_set(struct _pid *pid, int percent)
360 {
361 	pid->p_gain = div_fp(int_tofp(percent), int_tofp(100));
362 }
363 
364 static inline void pid_i_gain_set(struct _pid *pid, int percent)
365 {
366 	pid->i_gain = div_fp(int_tofp(percent), int_tofp(100));
367 }
368 
369 static inline void pid_d_gain_set(struct _pid *pid, int percent)
370 {
371 	pid->d_gain = div_fp(int_tofp(percent), int_tofp(100));
372 }
373 
374 static signed int pid_calc(struct _pid *pid, int32_t busy)
375 {
376 	signed int result;
377 	int32_t pterm, dterm, fp_error;
378 	int32_t integral_limit;
379 
380 	fp_error = int_tofp(pid->setpoint) - busy;
381 
382 	if (abs(fp_error) <= int_tofp(pid->deadband))
383 		return 0;
384 
385 	pterm = mul_fp(pid->p_gain, fp_error);
386 
387 	pid->integral += fp_error;
388 
389 	/*
390 	 * We limit the integral here so that it will never
391 	 * get higher than 30.  This prevents it from becoming
392 	 * too large an input over long periods of time and allows
393 	 * it to get factored out sooner.
394 	 *
395 	 * The value of 30 was chosen through experimentation.
396 	 */
397 	integral_limit = int_tofp(30);
398 	if (pid->integral > integral_limit)
399 		pid->integral = integral_limit;
400 	if (pid->integral < -integral_limit)
401 		pid->integral = -integral_limit;
402 
403 	dterm = mul_fp(pid->d_gain, fp_error - pid->last_err);
404 	pid->last_err = fp_error;
405 
406 	result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm;
407 	result = result + (1 << (FRAC_BITS-1));
408 	return (signed int)fp_toint(result);
409 }
410 
411 static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu)
412 {
413 	pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct);
414 	pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
415 	pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
416 
417 	pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0);
418 }
419 
420 static inline void intel_pstate_reset_all_pid(void)
421 {
422 	unsigned int cpu;
423 
424 	for_each_online_cpu(cpu) {
425 		if (all_cpu_data[cpu])
426 			intel_pstate_busy_pid_reset(all_cpu_data[cpu]);
427 	}
428 }
429 
430 static inline void update_turbo_state(void)
431 {
432 	u64 misc_en;
433 	struct cpudata *cpu;
434 
435 	cpu = all_cpu_data[0];
436 	rdmsrl(MSR_IA32_MISC_ENABLE, misc_en);
437 	limits->turbo_disabled =
438 		(misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ||
439 		 cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
440 }
441 
442 static void intel_pstate_hwp_set(void)
443 {
444 	int min, hw_min, max, hw_max, cpu, range, adj_range;
445 	u64 value, cap;
446 
447 	rdmsrl(MSR_HWP_CAPABILITIES, cap);
448 	hw_min = HWP_LOWEST_PERF(cap);
449 	hw_max = HWP_HIGHEST_PERF(cap);
450 	range = hw_max - hw_min;
451 
452 	get_online_cpus();
453 
454 	for_each_online_cpu(cpu) {
455 		rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
456 		adj_range = limits->min_perf_pct * range / 100;
457 		min = hw_min + adj_range;
458 		value &= ~HWP_MIN_PERF(~0L);
459 		value |= HWP_MIN_PERF(min);
460 
461 		adj_range = limits->max_perf_pct * range / 100;
462 		max = hw_min + adj_range;
463 		if (limits->no_turbo) {
464 			hw_max = HWP_GUARANTEED_PERF(cap);
465 			if (hw_max < max)
466 				max = hw_max;
467 		}
468 
469 		value &= ~HWP_MAX_PERF(~0L);
470 		value |= HWP_MAX_PERF(max);
471 		wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
472 	}
473 
474 	put_online_cpus();
475 }
476 
477 /************************** debugfs begin ************************/
478 static int pid_param_set(void *data, u64 val)
479 {
480 	*(u32 *)data = val;
481 	intel_pstate_reset_all_pid();
482 	return 0;
483 }
484 
485 static int pid_param_get(void *data, u64 *val)
486 {
487 	*val = *(u32 *)data;
488 	return 0;
489 }
490 DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get, pid_param_set, "%llu\n");
491 
492 struct pid_param {
493 	char *name;
494 	void *value;
495 };
496 
497 static struct pid_param pid_files[] = {
498 	{"sample_rate_ms", &pid_params.sample_rate_ms},
499 	{"d_gain_pct", &pid_params.d_gain_pct},
500 	{"i_gain_pct", &pid_params.i_gain_pct},
501 	{"deadband", &pid_params.deadband},
502 	{"setpoint", &pid_params.setpoint},
503 	{"p_gain_pct", &pid_params.p_gain_pct},
504 	{NULL, NULL}
505 };
506 
507 static void __init intel_pstate_debug_expose_params(void)
508 {
509 	struct dentry *debugfs_parent;
510 	int i = 0;
511 
512 	if (hwp_active)
513 		return;
514 	debugfs_parent = debugfs_create_dir("pstate_snb", NULL);
515 	if (IS_ERR_OR_NULL(debugfs_parent))
516 		return;
517 	while (pid_files[i].name) {
518 		debugfs_create_file(pid_files[i].name, 0660,
519 				    debugfs_parent, pid_files[i].value,
520 				    &fops_pid_param);
521 		i++;
522 	}
523 }
524 
525 /************************** debugfs end ************************/
526 
527 /************************** sysfs begin ************************/
528 #define show_one(file_name, object)					\
529 	static ssize_t show_##file_name					\
530 	(struct kobject *kobj, struct attribute *attr, char *buf)	\
531 	{								\
532 		return sprintf(buf, "%u\n", limits->object);		\
533 	}
534 
535 static ssize_t show_turbo_pct(struct kobject *kobj,
536 				struct attribute *attr, char *buf)
537 {
538 	struct cpudata *cpu;
539 	int total, no_turbo, turbo_pct;
540 	uint32_t turbo_fp;
541 
542 	cpu = all_cpu_data[0];
543 
544 	total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
545 	no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1;
546 	turbo_fp = div_fp(int_tofp(no_turbo), int_tofp(total));
547 	turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100)));
548 	return sprintf(buf, "%u\n", turbo_pct);
549 }
550 
551 static ssize_t show_num_pstates(struct kobject *kobj,
552 				struct attribute *attr, char *buf)
553 {
554 	struct cpudata *cpu;
555 	int total;
556 
557 	cpu = all_cpu_data[0];
558 	total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
559 	return sprintf(buf, "%u\n", total);
560 }
561 
562 static ssize_t show_no_turbo(struct kobject *kobj,
563 			     struct attribute *attr, char *buf)
564 {
565 	ssize_t ret;
566 
567 	update_turbo_state();
568 	if (limits->turbo_disabled)
569 		ret = sprintf(buf, "%u\n", limits->turbo_disabled);
570 	else
571 		ret = sprintf(buf, "%u\n", limits->no_turbo);
572 
573 	return ret;
574 }
575 
576 static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
577 			      const char *buf, size_t count)
578 {
579 	unsigned int input;
580 	int ret;
581 
582 	ret = sscanf(buf, "%u", &input);
583 	if (ret != 1)
584 		return -EINVAL;
585 
586 	update_turbo_state();
587 	if (limits->turbo_disabled) {
588 		pr_warn("intel_pstate: Turbo disabled by BIOS or unavailable on processor\n");
589 		return -EPERM;
590 	}
591 
592 	limits->no_turbo = clamp_t(int, input, 0, 1);
593 
594 	if (hwp_active)
595 		intel_pstate_hwp_set();
596 
597 	return count;
598 }
599 
600 static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
601 				  const char *buf, size_t count)
602 {
603 	unsigned int input;
604 	int ret;
605 
606 	ret = sscanf(buf, "%u", &input);
607 	if (ret != 1)
608 		return -EINVAL;
609 
610 	limits->max_sysfs_pct = clamp_t(int, input, 0 , 100);
611 	limits->max_perf_pct = min(limits->max_policy_pct,
612 				   limits->max_sysfs_pct);
613 	limits->max_perf_pct = max(limits->min_policy_pct,
614 				   limits->max_perf_pct);
615 	limits->max_perf_pct = max(limits->min_perf_pct,
616 				   limits->max_perf_pct);
617 	limits->max_perf = div_fp(int_tofp(limits->max_perf_pct),
618 				  int_tofp(100));
619 
620 	if (hwp_active)
621 		intel_pstate_hwp_set();
622 	return count;
623 }
624 
625 static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
626 				  const char *buf, size_t count)
627 {
628 	unsigned int input;
629 	int ret;
630 
631 	ret = sscanf(buf, "%u", &input);
632 	if (ret != 1)
633 		return -EINVAL;
634 
635 	limits->min_sysfs_pct = clamp_t(int, input, 0 , 100);
636 	limits->min_perf_pct = max(limits->min_policy_pct,
637 				   limits->min_sysfs_pct);
638 	limits->min_perf_pct = min(limits->max_policy_pct,
639 				   limits->min_perf_pct);
640 	limits->min_perf_pct = min(limits->max_perf_pct,
641 				   limits->min_perf_pct);
642 	limits->min_perf = div_fp(int_tofp(limits->min_perf_pct),
643 				  int_tofp(100));
644 
645 	if (hwp_active)
646 		intel_pstate_hwp_set();
647 	return count;
648 }
649 
650 show_one(max_perf_pct, max_perf_pct);
651 show_one(min_perf_pct, min_perf_pct);
652 
653 define_one_global_rw(no_turbo);
654 define_one_global_rw(max_perf_pct);
655 define_one_global_rw(min_perf_pct);
656 define_one_global_ro(turbo_pct);
657 define_one_global_ro(num_pstates);
658 
659 static struct attribute *intel_pstate_attributes[] = {
660 	&no_turbo.attr,
661 	&max_perf_pct.attr,
662 	&min_perf_pct.attr,
663 	&turbo_pct.attr,
664 	&num_pstates.attr,
665 	NULL
666 };
667 
668 static struct attribute_group intel_pstate_attr_group = {
669 	.attrs = intel_pstate_attributes,
670 };
671 
672 static void __init intel_pstate_sysfs_expose_params(void)
673 {
674 	struct kobject *intel_pstate_kobject;
675 	int rc;
676 
677 	intel_pstate_kobject = kobject_create_and_add("intel_pstate",
678 						&cpu_subsys.dev_root->kobj);
679 	BUG_ON(!intel_pstate_kobject);
680 	rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
681 	BUG_ON(rc);
682 }
683 /************************** sysfs end ************************/
684 
685 static void intel_pstate_hwp_enable(struct cpudata *cpudata)
686 {
687 	pr_info("intel_pstate: HWP enabled\n");
688 
689 	wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
690 }
691 
692 static int byt_get_min_pstate(void)
693 {
694 	u64 value;
695 
696 	rdmsrl(BYT_RATIOS, value);
697 	return (value >> 8) & 0x7F;
698 }
699 
700 static int byt_get_max_pstate(void)
701 {
702 	u64 value;
703 
704 	rdmsrl(BYT_RATIOS, value);
705 	return (value >> 16) & 0x7F;
706 }
707 
708 static int byt_get_turbo_pstate(void)
709 {
710 	u64 value;
711 
712 	rdmsrl(BYT_TURBO_RATIOS, value);
713 	return value & 0x7F;
714 }
715 
716 static void byt_set_pstate(struct cpudata *cpudata, int pstate)
717 {
718 	u64 val;
719 	int32_t vid_fp;
720 	u32 vid;
721 
722 	val = (u64)pstate << 8;
723 	if (limits->no_turbo && !limits->turbo_disabled)
724 		val |= (u64)1 << 32;
725 
726 	vid_fp = cpudata->vid.min + mul_fp(
727 		int_tofp(pstate - cpudata->pstate.min_pstate),
728 		cpudata->vid.ratio);
729 
730 	vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
731 	vid = ceiling_fp(vid_fp);
732 
733 	if (pstate > cpudata->pstate.max_pstate)
734 		vid = cpudata->vid.turbo;
735 
736 	val |= vid;
737 
738 	wrmsrl_on_cpu(cpudata->cpu, MSR_IA32_PERF_CTL, val);
739 }
740 
741 #define BYT_BCLK_FREQS 5
742 static int byt_freq_table[BYT_BCLK_FREQS] = { 833, 1000, 1333, 1167, 800};
743 
744 static int byt_get_scaling(void)
745 {
746 	u64 value;
747 	int i;
748 
749 	rdmsrl(MSR_FSB_FREQ, value);
750 	i = value & 0x3;
751 
752 	BUG_ON(i > BYT_BCLK_FREQS);
753 
754 	return byt_freq_table[i] * 100;
755 }
756 
757 static void byt_get_vid(struct cpudata *cpudata)
758 {
759 	u64 value;
760 
761 	rdmsrl(BYT_VIDS, value);
762 	cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
763 	cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
764 	cpudata->vid.ratio = div_fp(
765 		cpudata->vid.max - cpudata->vid.min,
766 		int_tofp(cpudata->pstate.max_pstate -
767 			cpudata->pstate.min_pstate));
768 
769 	rdmsrl(BYT_TURBO_VIDS, value);
770 	cpudata->vid.turbo = value & 0x7f;
771 }
772 
773 static int core_get_min_pstate(void)
774 {
775 	u64 value;
776 
777 	rdmsrl(MSR_PLATFORM_INFO, value);
778 	return (value >> 40) & 0xFF;
779 }
780 
781 static int core_get_max_pstate_physical(void)
782 {
783 	u64 value;
784 
785 	rdmsrl(MSR_PLATFORM_INFO, value);
786 	return (value >> 8) & 0xFF;
787 }
788 
789 static int core_get_max_pstate(void)
790 {
791 	u64 tar;
792 	u64 plat_info;
793 	int max_pstate;
794 	int err;
795 
796 	rdmsrl(MSR_PLATFORM_INFO, plat_info);
797 	max_pstate = (plat_info >> 8) & 0xFF;
798 
799 	err = rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO, &tar);
800 	if (!err) {
801 		/* Do some sanity checking for safety */
802 		if (plat_info & 0x600000000) {
803 			u64 tdp_ctrl;
804 			u64 tdp_ratio;
805 			int tdp_msr;
806 
807 			err = rdmsrl_safe(MSR_CONFIG_TDP_CONTROL, &tdp_ctrl);
808 			if (err)
809 				goto skip_tar;
810 
811 			tdp_msr = MSR_CONFIG_TDP_NOMINAL + tdp_ctrl;
812 			err = rdmsrl_safe(tdp_msr, &tdp_ratio);
813 			if (err)
814 				goto skip_tar;
815 
816 			if (tdp_ratio - 1 == tar) {
817 				max_pstate = tar;
818 				pr_debug("max_pstate=TAC %x\n", max_pstate);
819 			} else {
820 				goto skip_tar;
821 			}
822 		}
823 	}
824 
825 skip_tar:
826 	return max_pstate;
827 }
828 
829 static int core_get_turbo_pstate(void)
830 {
831 	u64 value;
832 	int nont, ret;
833 
834 	rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value);
835 	nont = core_get_max_pstate();
836 	ret = (value) & 255;
837 	if (ret <= nont)
838 		ret = nont;
839 	return ret;
840 }
841 
842 static inline int core_get_scaling(void)
843 {
844 	return 100000;
845 }
846 
847 static void core_set_pstate(struct cpudata *cpudata, int pstate)
848 {
849 	u64 val;
850 
851 	val = (u64)pstate << 8;
852 	if (limits->no_turbo && !limits->turbo_disabled)
853 		val |= (u64)1 << 32;
854 
855 	wrmsrl_on_cpu(cpudata->cpu, MSR_IA32_PERF_CTL, val);
856 }
857 
858 static int knl_get_turbo_pstate(void)
859 {
860 	u64 value;
861 	int nont, ret;
862 
863 	rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT, value);
864 	nont = core_get_max_pstate();
865 	ret = (((value) >> 8) & 0xFF);
866 	if (ret <= nont)
867 		ret = nont;
868 	return ret;
869 }
870 
871 static struct cpu_defaults core_params = {
872 	.pid_policy = {
873 		.sample_rate_ms = 10,
874 		.deadband = 0,
875 		.setpoint = 97,
876 		.p_gain_pct = 20,
877 		.d_gain_pct = 0,
878 		.i_gain_pct = 0,
879 	},
880 	.funcs = {
881 		.get_max = core_get_max_pstate,
882 		.get_max_physical = core_get_max_pstate_physical,
883 		.get_min = core_get_min_pstate,
884 		.get_turbo = core_get_turbo_pstate,
885 		.get_scaling = core_get_scaling,
886 		.set = core_set_pstate,
887 	},
888 };
889 
890 static struct cpu_defaults byt_params = {
891 	.pid_policy = {
892 		.sample_rate_ms = 10,
893 		.deadband = 0,
894 		.setpoint = 60,
895 		.p_gain_pct = 14,
896 		.d_gain_pct = 0,
897 		.i_gain_pct = 4,
898 	},
899 	.funcs = {
900 		.get_max = byt_get_max_pstate,
901 		.get_max_physical = byt_get_max_pstate,
902 		.get_min = byt_get_min_pstate,
903 		.get_turbo = byt_get_turbo_pstate,
904 		.set = byt_set_pstate,
905 		.get_scaling = byt_get_scaling,
906 		.get_vid = byt_get_vid,
907 	},
908 };
909 
910 static struct cpu_defaults knl_params = {
911 	.pid_policy = {
912 		.sample_rate_ms = 10,
913 		.deadband = 0,
914 		.setpoint = 97,
915 		.p_gain_pct = 20,
916 		.d_gain_pct = 0,
917 		.i_gain_pct = 0,
918 	},
919 	.funcs = {
920 		.get_max = core_get_max_pstate,
921 		.get_max_physical = core_get_max_pstate_physical,
922 		.get_min = core_get_min_pstate,
923 		.get_turbo = knl_get_turbo_pstate,
924 		.get_scaling = core_get_scaling,
925 		.set = core_set_pstate,
926 	},
927 };
928 
929 static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
930 {
931 	int max_perf = cpu->pstate.turbo_pstate;
932 	int max_perf_adj;
933 	int min_perf;
934 
935 	if (limits->no_turbo || limits->turbo_disabled)
936 		max_perf = cpu->pstate.max_pstate;
937 
938 	/*
939 	 * performance can be limited by user through sysfs, by cpufreq
940 	 * policy, or by cpu specific default values determined through
941 	 * experimentation.
942 	 */
943 	if (limits->max_perf_ctl && limits->max_sysfs_pct >=
944 						limits->max_policy_pct) {
945 		*max = limits->max_perf_ctl;
946 	} else {
947 		max_perf_adj = fp_toint(mul_fp(int_tofp(max_perf),
948 					limits->max_perf));
949 		*max = clamp_t(int, max_perf_adj, cpu->pstate.min_pstate,
950 			       cpu->pstate.turbo_pstate);
951 	}
952 
953 	if (limits->min_perf_ctl) {
954 		*min = limits->min_perf_ctl;
955 	} else {
956 		min_perf = fp_toint(mul_fp(int_tofp(max_perf),
957 				    limits->min_perf));
958 		*min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
959 	}
960 }
961 
962 static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate, bool force)
963 {
964 	int max_perf, min_perf;
965 
966 	if (force) {
967 		update_turbo_state();
968 
969 		intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
970 
971 		pstate = clamp_t(int, pstate, min_perf, max_perf);
972 
973 		if (pstate == cpu->pstate.current_pstate)
974 			return;
975 	}
976 	trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
977 
978 	cpu->pstate.current_pstate = pstate;
979 
980 	pstate_funcs.set(cpu, pstate);
981 }
982 
983 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
984 {
985 	cpu->pstate.min_pstate = pstate_funcs.get_min();
986 	cpu->pstate.max_pstate = pstate_funcs.get_max();
987 	cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical();
988 	cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
989 	cpu->pstate.scaling = pstate_funcs.get_scaling();
990 
991 	if (pstate_funcs.get_vid)
992 		pstate_funcs.get_vid(cpu);
993 	intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate, false);
994 }
995 
996 static inline void intel_pstate_calc_busy(struct cpudata *cpu)
997 {
998 	struct sample *sample = &cpu->sample;
999 	int64_t core_pct;
1000 
1001 	core_pct = int_tofp(sample->aperf) * int_tofp(100);
1002 	core_pct = div64_u64(core_pct, int_tofp(sample->mperf));
1003 
1004 	sample->freq = fp_toint(
1005 		mul_fp(int_tofp(
1006 			cpu->pstate.max_pstate_physical *
1007 			cpu->pstate.scaling / 100),
1008 			core_pct));
1009 
1010 	sample->core_pct_busy = (int32_t)core_pct;
1011 }
1012 
1013 static inline void intel_pstate_sample(struct cpudata *cpu)
1014 {
1015 	u64 aperf, mperf;
1016 	unsigned long flags;
1017 	u64 tsc;
1018 
1019 	local_irq_save(flags);
1020 	rdmsrl(MSR_IA32_APERF, aperf);
1021 	rdmsrl(MSR_IA32_MPERF, mperf);
1022 	if (cpu->prev_mperf == mperf) {
1023 		local_irq_restore(flags);
1024 		return;
1025 	}
1026 
1027 	tsc = rdtsc();
1028 	local_irq_restore(flags);
1029 
1030 	cpu->last_sample_time = cpu->sample.time;
1031 	cpu->sample.time = ktime_get();
1032 	cpu->sample.aperf = aperf;
1033 	cpu->sample.mperf = mperf;
1034 	cpu->sample.tsc =  tsc;
1035 	cpu->sample.aperf -= cpu->prev_aperf;
1036 	cpu->sample.mperf -= cpu->prev_mperf;
1037 	cpu->sample.tsc -= cpu->prev_tsc;
1038 
1039 	intel_pstate_calc_busy(cpu);
1040 
1041 	cpu->prev_aperf = aperf;
1042 	cpu->prev_mperf = mperf;
1043 	cpu->prev_tsc = tsc;
1044 }
1045 
1046 static inline void intel_hwp_set_sample_time(struct cpudata *cpu)
1047 {
1048 	int delay;
1049 
1050 	delay = msecs_to_jiffies(50);
1051 	mod_timer_pinned(&cpu->timer, jiffies + delay);
1052 }
1053 
1054 static inline void intel_pstate_set_sample_time(struct cpudata *cpu)
1055 {
1056 	int delay;
1057 
1058 	delay = msecs_to_jiffies(pid_params.sample_rate_ms);
1059 	mod_timer_pinned(&cpu->timer, jiffies + delay);
1060 }
1061 
1062 static inline int32_t intel_pstate_get_scaled_busy(struct cpudata *cpu)
1063 {
1064 	int32_t core_busy, max_pstate, current_pstate, sample_ratio;
1065 	s64 duration_us;
1066 	u32 sample_time;
1067 
1068 	/*
1069 	 * core_busy is the ratio of actual performance to max
1070 	 * max_pstate is the max non turbo pstate available
1071 	 * current_pstate was the pstate that was requested during
1072 	 * 	the last sample period.
1073 	 *
1074 	 * We normalize core_busy, which was our actual percent
1075 	 * performance to what we requested during the last sample
1076 	 * period. The result will be a percentage of busy at a
1077 	 * specified pstate.
1078 	 */
1079 	core_busy = cpu->sample.core_pct_busy;
1080 	max_pstate = int_tofp(cpu->pstate.max_pstate_physical);
1081 	current_pstate = int_tofp(cpu->pstate.current_pstate);
1082 	core_busy = mul_fp(core_busy, div_fp(max_pstate, current_pstate));
1083 
1084 	/*
1085 	 * Since we have a deferred timer, it will not fire unless
1086 	 * we are in C0.  So, determine if the actual elapsed time
1087 	 * is significantly greater (3x) than our sample interval.  If it
1088 	 * is, then we were idle for a long enough period of time
1089 	 * to adjust our busyness.
1090 	 */
1091 	sample_time = pid_params.sample_rate_ms  * USEC_PER_MSEC;
1092 	duration_us = ktime_us_delta(cpu->sample.time,
1093 				     cpu->last_sample_time);
1094 	if (duration_us > sample_time * 3) {
1095 		sample_ratio = div_fp(int_tofp(sample_time),
1096 				      int_tofp(duration_us));
1097 		core_busy = mul_fp(core_busy, sample_ratio);
1098 	}
1099 
1100 	return core_busy;
1101 }
1102 
1103 static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
1104 {
1105 	int32_t busy_scaled;
1106 	struct _pid *pid;
1107 	signed int ctl;
1108 	int from;
1109 	struct sample *sample;
1110 
1111 	from = cpu->pstate.current_pstate;
1112 
1113 	pid = &cpu->pid;
1114 	busy_scaled = intel_pstate_get_scaled_busy(cpu);
1115 
1116 	ctl = pid_calc(pid, busy_scaled);
1117 
1118 	/* Negative values of ctl increase the pstate and vice versa */
1119 	intel_pstate_set_pstate(cpu, cpu->pstate.current_pstate - ctl, true);
1120 
1121 	sample = &cpu->sample;
1122 	trace_pstate_sample(fp_toint(sample->core_pct_busy),
1123 		fp_toint(busy_scaled),
1124 		from,
1125 		cpu->pstate.current_pstate,
1126 		sample->mperf,
1127 		sample->aperf,
1128 		sample->tsc,
1129 		sample->freq);
1130 }
1131 
1132 static void intel_hwp_timer_func(unsigned long __data)
1133 {
1134 	struct cpudata *cpu = (struct cpudata *) __data;
1135 
1136 	intel_pstate_sample(cpu);
1137 	intel_hwp_set_sample_time(cpu);
1138 }
1139 
1140 static void intel_pstate_timer_func(unsigned long __data)
1141 {
1142 	struct cpudata *cpu = (struct cpudata *) __data;
1143 
1144 	intel_pstate_sample(cpu);
1145 
1146 	intel_pstate_adjust_busy_pstate(cpu);
1147 
1148 	intel_pstate_set_sample_time(cpu);
1149 }
1150 
1151 #define ICPU(model, policy) \
1152 	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
1153 			(unsigned long)&policy }
1154 
1155 static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
1156 	ICPU(0x2a, core_params),
1157 	ICPU(0x2d, core_params),
1158 	ICPU(0x37, byt_params),
1159 	ICPU(0x3a, core_params),
1160 	ICPU(0x3c, core_params),
1161 	ICPU(0x3d, core_params),
1162 	ICPU(0x3e, core_params),
1163 	ICPU(0x3f, core_params),
1164 	ICPU(0x45, core_params),
1165 	ICPU(0x46, core_params),
1166 	ICPU(0x47, core_params),
1167 	ICPU(0x4c, byt_params),
1168 	ICPU(0x4e, core_params),
1169 	ICPU(0x4f, core_params),
1170 	ICPU(0x5e, core_params),
1171 	ICPU(0x56, core_params),
1172 	ICPU(0x57, knl_params),
1173 	{}
1174 };
1175 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
1176 
1177 static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] = {
1178 	ICPU(0x56, core_params),
1179 	{}
1180 };
1181 
1182 static int intel_pstate_init_cpu(unsigned int cpunum)
1183 {
1184 	struct cpudata *cpu;
1185 
1186 	if (!all_cpu_data[cpunum])
1187 		all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata),
1188 					       GFP_KERNEL);
1189 	if (!all_cpu_data[cpunum])
1190 		return -ENOMEM;
1191 
1192 	cpu = all_cpu_data[cpunum];
1193 
1194 	cpu->cpu = cpunum;
1195 
1196 	if (hwp_active)
1197 		intel_pstate_hwp_enable(cpu);
1198 
1199 	intel_pstate_get_cpu_pstates(cpu);
1200 
1201 	init_timer_deferrable(&cpu->timer);
1202 	cpu->timer.data = (unsigned long)cpu;
1203 	cpu->timer.expires = jiffies + HZ/100;
1204 
1205 	if (!hwp_active)
1206 		cpu->timer.function = intel_pstate_timer_func;
1207 	else
1208 		cpu->timer.function = intel_hwp_timer_func;
1209 
1210 	intel_pstate_busy_pid_reset(cpu);
1211 	intel_pstate_sample(cpu);
1212 
1213 	add_timer_on(&cpu->timer, cpunum);
1214 
1215 	pr_debug("intel_pstate: controlling: cpu %d\n", cpunum);
1216 
1217 	return 0;
1218 }
1219 
1220 static unsigned int intel_pstate_get(unsigned int cpu_num)
1221 {
1222 	struct sample *sample;
1223 	struct cpudata *cpu;
1224 
1225 	cpu = all_cpu_data[cpu_num];
1226 	if (!cpu)
1227 		return 0;
1228 	sample = &cpu->sample;
1229 	return sample->freq;
1230 }
1231 
1232 static int intel_pstate_set_policy(struct cpufreq_policy *policy)
1233 {
1234 #if IS_ENABLED(CONFIG_ACPI)
1235 	struct cpudata *cpu;
1236 	int i;
1237 #endif
1238 	pr_debug("intel_pstate: %s max %u policy->max %u\n", __func__,
1239 		 policy->cpuinfo.max_freq, policy->max);
1240 	if (!policy->cpuinfo.max_freq)
1241 		return -ENODEV;
1242 
1243 	if (policy->policy == CPUFREQ_POLICY_PERFORMANCE &&
1244 	    policy->max >= policy->cpuinfo.max_freq) {
1245 		pr_debug("intel_pstate: set performance\n");
1246 		limits = &performance_limits;
1247 		return 0;
1248 	}
1249 
1250 	pr_debug("intel_pstate: set powersave\n");
1251 	limits = &powersave_limits;
1252 	limits->min_policy_pct = (policy->min * 100) / policy->cpuinfo.max_freq;
1253 	limits->min_policy_pct = clamp_t(int, limits->min_policy_pct, 0 , 100);
1254 	limits->max_policy_pct = (policy->max * 100) / policy->cpuinfo.max_freq;
1255 	limits->max_policy_pct = clamp_t(int, limits->max_policy_pct, 0 , 100);
1256 
1257 	/* Normalize user input to [min_policy_pct, max_policy_pct] */
1258 	limits->min_perf_pct = max(limits->min_policy_pct,
1259 				   limits->min_sysfs_pct);
1260 	limits->min_perf_pct = min(limits->max_policy_pct,
1261 				   limits->min_perf_pct);
1262 	limits->max_perf_pct = min(limits->max_policy_pct,
1263 				   limits->max_sysfs_pct);
1264 	limits->max_perf_pct = max(limits->min_policy_pct,
1265 				   limits->max_perf_pct);
1266 
1267 	/* Make sure min_perf_pct <= max_perf_pct */
1268 	limits->min_perf_pct = min(limits->max_perf_pct, limits->min_perf_pct);
1269 
1270 	limits->min_perf = div_fp(int_tofp(limits->min_perf_pct),
1271 				  int_tofp(100));
1272 	limits->max_perf = div_fp(int_tofp(limits->max_perf_pct),
1273 				  int_tofp(100));
1274 
1275 #if IS_ENABLED(CONFIG_ACPI)
1276 	cpu = all_cpu_data[policy->cpu];
1277 	for (i = 0; i < cpu->acpi_perf_data.state_count; i++) {
1278 		int control;
1279 
1280 		control = convert_to_native_pstate_format(cpu, i);
1281 		if (control * cpu->pstate.scaling == policy->max)
1282 			limits->max_perf_ctl = control;
1283 		if (control * cpu->pstate.scaling == policy->min)
1284 			limits->min_perf_ctl = control;
1285 	}
1286 
1287 	pr_debug("intel_pstate: max %u policy_max %u perf_ctl [0x%x-0x%x]\n",
1288 		 policy->cpuinfo.max_freq, policy->max, limits->min_perf_ctl,
1289 		 limits->max_perf_ctl);
1290 #endif
1291 
1292 	if (hwp_active)
1293 		intel_pstate_hwp_set();
1294 
1295 	return 0;
1296 }
1297 
1298 static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
1299 {
1300 	cpufreq_verify_within_cpu_limits(policy);
1301 
1302 	if (policy->policy != CPUFREQ_POLICY_POWERSAVE &&
1303 	    policy->policy != CPUFREQ_POLICY_PERFORMANCE)
1304 		return -EINVAL;
1305 
1306 	return 0;
1307 }
1308 
1309 static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
1310 {
1311 	int cpu_num = policy->cpu;
1312 	struct cpudata *cpu = all_cpu_data[cpu_num];
1313 
1314 	pr_debug("intel_pstate: CPU %d exiting\n", cpu_num);
1315 
1316 	del_timer_sync(&all_cpu_data[cpu_num]->timer);
1317 	if (hwp_active)
1318 		return;
1319 
1320 	intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate, false);
1321 }
1322 
1323 static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
1324 {
1325 	struct cpudata *cpu;
1326 	int rc;
1327 
1328 	rc = intel_pstate_init_cpu(policy->cpu);
1329 	if (rc)
1330 		return rc;
1331 
1332 	cpu = all_cpu_data[policy->cpu];
1333 
1334 	if (limits->min_perf_pct == 100 && limits->max_perf_pct == 100)
1335 		policy->policy = CPUFREQ_POLICY_PERFORMANCE;
1336 	else
1337 		policy->policy = CPUFREQ_POLICY_POWERSAVE;
1338 
1339 	policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
1340 	policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
1341 
1342 	/* cpuinfo and default policy values */
1343 	policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling;
1344 	policy->cpuinfo.max_freq =
1345 		cpu->pstate.turbo_pstate * cpu->pstate.scaling;
1346 	if (!no_acpi_perf)
1347 		intel_pstate_init_perf_limits(policy);
1348 	/*
1349 	 * If there is no acpi perf data or error, we ignore and use Intel P
1350 	 * state calculated limits, So this is not fatal error.
1351 	 */
1352 	policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
1353 	cpumask_set_cpu(policy->cpu, policy->cpus);
1354 
1355 	return 0;
1356 }
1357 
1358 static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
1359 {
1360 	return intel_pstate_exit_perf_limits(policy);
1361 }
1362 
1363 static struct cpufreq_driver intel_pstate_driver = {
1364 	.flags		= CPUFREQ_CONST_LOOPS,
1365 	.verify		= intel_pstate_verify_policy,
1366 	.setpolicy	= intel_pstate_set_policy,
1367 	.get		= intel_pstate_get,
1368 	.init		= intel_pstate_cpu_init,
1369 	.exit		= intel_pstate_cpu_exit,
1370 	.stop_cpu	= intel_pstate_stop_cpu,
1371 	.name		= "intel_pstate",
1372 };
1373 
1374 static int __initdata no_load;
1375 static int __initdata no_hwp;
1376 static int __initdata hwp_only;
1377 static unsigned int force_load;
1378 
1379 static int intel_pstate_msrs_not_valid(void)
1380 {
1381 	if (!pstate_funcs.get_max() ||
1382 	    !pstate_funcs.get_min() ||
1383 	    !pstate_funcs.get_turbo())
1384 		return -ENODEV;
1385 
1386 	return 0;
1387 }
1388 
1389 static void copy_pid_params(struct pstate_adjust_policy *policy)
1390 {
1391 	pid_params.sample_rate_ms = policy->sample_rate_ms;
1392 	pid_params.p_gain_pct = policy->p_gain_pct;
1393 	pid_params.i_gain_pct = policy->i_gain_pct;
1394 	pid_params.d_gain_pct = policy->d_gain_pct;
1395 	pid_params.deadband = policy->deadband;
1396 	pid_params.setpoint = policy->setpoint;
1397 }
1398 
1399 static void copy_cpu_funcs(struct pstate_funcs *funcs)
1400 {
1401 	pstate_funcs.get_max   = funcs->get_max;
1402 	pstate_funcs.get_max_physical = funcs->get_max_physical;
1403 	pstate_funcs.get_min   = funcs->get_min;
1404 	pstate_funcs.get_turbo = funcs->get_turbo;
1405 	pstate_funcs.get_scaling = funcs->get_scaling;
1406 	pstate_funcs.set       = funcs->set;
1407 	pstate_funcs.get_vid   = funcs->get_vid;
1408 }
1409 
1410 #if IS_ENABLED(CONFIG_ACPI)
1411 
1412 static bool intel_pstate_no_acpi_pss(void)
1413 {
1414 	int i;
1415 
1416 	for_each_possible_cpu(i) {
1417 		acpi_status status;
1418 		union acpi_object *pss;
1419 		struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
1420 		struct acpi_processor *pr = per_cpu(processors, i);
1421 
1422 		if (!pr)
1423 			continue;
1424 
1425 		status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
1426 		if (ACPI_FAILURE(status))
1427 			continue;
1428 
1429 		pss = buffer.pointer;
1430 		if (pss && pss->type == ACPI_TYPE_PACKAGE) {
1431 			kfree(pss);
1432 			return false;
1433 		}
1434 
1435 		kfree(pss);
1436 	}
1437 
1438 	return true;
1439 }
1440 
1441 static bool intel_pstate_has_acpi_ppc(void)
1442 {
1443 	int i;
1444 
1445 	for_each_possible_cpu(i) {
1446 		struct acpi_processor *pr = per_cpu(processors, i);
1447 
1448 		if (!pr)
1449 			continue;
1450 		if (acpi_has_method(pr->handle, "_PPC"))
1451 			return true;
1452 	}
1453 	return false;
1454 }
1455 
1456 enum {
1457 	PSS,
1458 	PPC,
1459 };
1460 
1461 struct hw_vendor_info {
1462 	u16  valid;
1463 	char oem_id[ACPI_OEM_ID_SIZE];
1464 	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE];
1465 	int  oem_pwr_table;
1466 };
1467 
1468 /* Hardware vendor-specific info that has its own power management modes */
1469 static struct hw_vendor_info vendor_info[] = {
1470 	{1, "HP    ", "ProLiant", PSS},
1471 	{1, "ORACLE", "X4-2    ", PPC},
1472 	{1, "ORACLE", "X4-2L   ", PPC},
1473 	{1, "ORACLE", "X4-2B   ", PPC},
1474 	{1, "ORACLE", "X3-2    ", PPC},
1475 	{1, "ORACLE", "X3-2L   ", PPC},
1476 	{1, "ORACLE", "X3-2B   ", PPC},
1477 	{1, "ORACLE", "X4470M2 ", PPC},
1478 	{1, "ORACLE", "X4270M3 ", PPC},
1479 	{1, "ORACLE", "X4270M2 ", PPC},
1480 	{1, "ORACLE", "X4170M2 ", PPC},
1481 	{1, "ORACLE", "X4170 M3", PPC},
1482 	{1, "ORACLE", "X4275 M3", PPC},
1483 	{1, "ORACLE", "X6-2    ", PPC},
1484 	{1, "ORACLE", "Sudbury ", PPC},
1485 	{0, "", ""},
1486 };
1487 
1488 static bool intel_pstate_platform_pwr_mgmt_exists(void)
1489 {
1490 	struct acpi_table_header hdr;
1491 	struct hw_vendor_info *v_info;
1492 	const struct x86_cpu_id *id;
1493 	u64 misc_pwr;
1494 
1495 	id = x86_match_cpu(intel_pstate_cpu_oob_ids);
1496 	if (id) {
1497 		rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr);
1498 		if ( misc_pwr & (1 << 8))
1499 			return true;
1500 	}
1501 
1502 	if (acpi_disabled ||
1503 	    ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr)))
1504 		return false;
1505 
1506 	for (v_info = vendor_info; v_info->valid; v_info++) {
1507 		if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE) &&
1508 			!strncmp(hdr.oem_table_id, v_info->oem_table_id,
1509 						ACPI_OEM_TABLE_ID_SIZE))
1510 			switch (v_info->oem_pwr_table) {
1511 			case PSS:
1512 				return intel_pstate_no_acpi_pss();
1513 			case PPC:
1514 				return intel_pstate_has_acpi_ppc() &&
1515 					(!force_load);
1516 			}
1517 	}
1518 
1519 	return false;
1520 }
1521 #else /* CONFIG_ACPI not enabled */
1522 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
1523 static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
1524 #endif /* CONFIG_ACPI */
1525 
1526 static int __init intel_pstate_init(void)
1527 {
1528 	int cpu, rc = 0;
1529 	const struct x86_cpu_id *id;
1530 	struct cpu_defaults *cpu_def;
1531 
1532 	if (no_load)
1533 		return -ENODEV;
1534 
1535 	id = x86_match_cpu(intel_pstate_cpu_ids);
1536 	if (!id)
1537 		return -ENODEV;
1538 
1539 	/*
1540 	 * The Intel pstate driver will be ignored if the platform
1541 	 * firmware has its own power management modes.
1542 	 */
1543 	if (intel_pstate_platform_pwr_mgmt_exists())
1544 		return -ENODEV;
1545 
1546 	cpu_def = (struct cpu_defaults *)id->driver_data;
1547 
1548 	copy_pid_params(&cpu_def->pid_policy);
1549 	copy_cpu_funcs(&cpu_def->funcs);
1550 
1551 	if (intel_pstate_msrs_not_valid())
1552 		return -ENODEV;
1553 
1554 	pr_info("Intel P-state driver initializing.\n");
1555 
1556 	all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus());
1557 	if (!all_cpu_data)
1558 		return -ENOMEM;
1559 
1560 	if (static_cpu_has_safe(X86_FEATURE_HWP) && !no_hwp)
1561 		hwp_active++;
1562 
1563 	if (!hwp_active && hwp_only)
1564 		goto out;
1565 
1566 	rc = cpufreq_register_driver(&intel_pstate_driver);
1567 	if (rc)
1568 		goto out;
1569 
1570 	intel_pstate_debug_expose_params();
1571 	intel_pstate_sysfs_expose_params();
1572 
1573 	return rc;
1574 out:
1575 	get_online_cpus();
1576 	for_each_online_cpu(cpu) {
1577 		if (all_cpu_data[cpu]) {
1578 			del_timer_sync(&all_cpu_data[cpu]->timer);
1579 			kfree(all_cpu_data[cpu]);
1580 		}
1581 	}
1582 
1583 	put_online_cpus();
1584 	vfree(all_cpu_data);
1585 	return -ENODEV;
1586 }
1587 device_initcall(intel_pstate_init);
1588 
1589 static int __init intel_pstate_setup(char *str)
1590 {
1591 	if (!str)
1592 		return -EINVAL;
1593 
1594 	if (!strcmp(str, "disable"))
1595 		no_load = 1;
1596 	if (!strcmp(str, "no_hwp"))
1597 		no_hwp = 1;
1598 	if (!strcmp(str, "force"))
1599 		force_load = 1;
1600 	if (!strcmp(str, "hwp_only"))
1601 		hwp_only = 1;
1602 	if (!strcmp(str, "no_acpi"))
1603 		no_acpi_perf = 1;
1604 
1605 	return 0;
1606 }
1607 early_param("intel_pstate", intel_pstate_setup);
1608 
1609 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
1610 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
1611 MODULE_LICENSE("GPL");
1612