1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * amd-pstate.c - AMD Processor P-state Frequency Driver
4 *
5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
6 *
7 * Author: Huang Rui <ray.huang@amd.com>
8 *
9 * AMD P-State introduces a new CPU performance scaling design for AMD
10 * processors using the ACPI Collaborative Performance and Power Control (CPPC)
11 * feature which works with the AMD SMU firmware providing a finer grained
12 * frequency control range. It is to replace the legacy ACPI P-States control,
13 * allows a flexible, low-latency interface for the Linux kernel to directly
14 * communicate the performance hints to hardware.
15 *
16 * AMD P-State is supported on recent AMD Zen base CPU series include some of
17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD
18 * P-State supported system. And there are two types of hardware implementations
19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types.
21 */
22
23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
24
25 #include <linux/kernel.h>
26 #include <linux/module.h>
27 #include <linux/init.h>
28 #include <linux/smp.h>
29 #include <linux/sched.h>
30 #include <linux/cpufreq.h>
31 #include <linux/compiler.h>
32 #include <linux/dmi.h>
33 #include <linux/slab.h>
34 #include <linux/acpi.h>
35 #include <linux/io.h>
36 #include <linux/delay.h>
37 #include <linux/uaccess.h>
38 #include <linux/static_call.h>
39 #include <linux/amd-pstate.h>
40 #include <linux/topology.h>
41
42 #include <acpi/processor.h>
43 #include <acpi/cppc_acpi.h>
44
45 #include <asm/msr.h>
46 #include <asm/processor.h>
47 #include <asm/cpufeature.h>
48 #include <asm/cpu_device_id.h>
49 #include "amd-pstate-trace.h"
50
51 #define AMD_PSTATE_TRANSITION_LATENCY 20000
52 #define AMD_PSTATE_TRANSITION_DELAY 1000
53 #define CPPC_HIGHEST_PERF_PERFORMANCE 196
54 #define CPPC_HIGHEST_PERF_DEFAULT 166
55
56 /*
57 * TODO: We need more time to fine tune processors with shared memory solution
58 * with community together.
59 *
60 * There are some performance drops on the CPU benchmarks which reports from
61 * Suse. We are co-working with them to fine tune the shared memory solution. So
62 * we disable it by default to go acpi-cpufreq on these processors and add a
63 * module parameter to be able to enable it manually for debugging.
64 */
65 static struct cpufreq_driver *current_pstate_driver;
66 static struct cpufreq_driver amd_pstate_driver;
67 static struct cpufreq_driver amd_pstate_epp_driver;
68 static int cppc_state = AMD_PSTATE_UNDEFINED;
69 static bool cppc_enabled;
70 static bool amd_pstate_prefcore = true;
71
72 /*
73 * AMD Energy Preference Performance (EPP)
74 * The EPP is used in the CCLK DPM controller to drive
75 * the frequency that a core is going to operate during
76 * short periods of activity. EPP values will be utilized for
77 * different OS profiles (balanced, performance, power savings)
78 * display strings corresponding to EPP index in the
79 * energy_perf_strings[]
80 * index String
81 *-------------------------------------
82 * 0 default
83 * 1 performance
84 * 2 balance_performance
85 * 3 balance_power
86 * 4 power
87 */
88 enum energy_perf_value_index {
89 EPP_INDEX_DEFAULT = 0,
90 EPP_INDEX_PERFORMANCE,
91 EPP_INDEX_BALANCE_PERFORMANCE,
92 EPP_INDEX_BALANCE_POWERSAVE,
93 EPP_INDEX_POWERSAVE,
94 };
95
96 static const char * const energy_perf_strings[] = {
97 [EPP_INDEX_DEFAULT] = "default",
98 [EPP_INDEX_PERFORMANCE] = "performance",
99 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance",
100 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power",
101 [EPP_INDEX_POWERSAVE] = "power",
102 NULL
103 };
104
105 static unsigned int epp_values[] = {
106 [EPP_INDEX_DEFAULT] = 0,
107 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE,
108 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE,
109 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE,
110 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE,
111 };
112
113 typedef int (*cppc_mode_transition_fn)(int);
114
get_mode_idx_from_str(const char * str,size_t size)115 static inline int get_mode_idx_from_str(const char *str, size_t size)
116 {
117 int i;
118
119 for (i=0; i < AMD_PSTATE_MAX; i++) {
120 if (!strncmp(str, amd_pstate_mode_string[i], size))
121 return i;
122 }
123 return -EINVAL;
124 }
125
126 static DEFINE_MUTEX(amd_pstate_limits_lock);
127 static DEFINE_MUTEX(amd_pstate_driver_lock);
128
amd_pstate_get_epp(struct amd_cpudata * cpudata,u64 cppc_req_cached)129 static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached)
130 {
131 u64 epp;
132 int ret;
133
134 if (boot_cpu_has(X86_FEATURE_CPPC)) {
135 if (!cppc_req_cached) {
136 epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
137 &cppc_req_cached);
138 if (epp)
139 return epp;
140 }
141 epp = (cppc_req_cached >> 24) & 0xFF;
142 } else {
143 ret = cppc_get_epp_perf(cpudata->cpu, &epp);
144 if (ret < 0) {
145 pr_debug("Could not retrieve energy perf value (%d)\n", ret);
146 return -EIO;
147 }
148 }
149
150 return (s16)(epp & 0xff);
151 }
152
amd_pstate_get_energy_pref_index(struct amd_cpudata * cpudata)153 static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata)
154 {
155 s16 epp;
156 int index = -EINVAL;
157
158 epp = amd_pstate_get_epp(cpudata, 0);
159 if (epp < 0)
160 return epp;
161
162 switch (epp) {
163 case AMD_CPPC_EPP_PERFORMANCE:
164 index = EPP_INDEX_PERFORMANCE;
165 break;
166 case AMD_CPPC_EPP_BALANCE_PERFORMANCE:
167 index = EPP_INDEX_BALANCE_PERFORMANCE;
168 break;
169 case AMD_CPPC_EPP_BALANCE_POWERSAVE:
170 index = EPP_INDEX_BALANCE_POWERSAVE;
171 break;
172 case AMD_CPPC_EPP_POWERSAVE:
173 index = EPP_INDEX_POWERSAVE;
174 break;
175 default:
176 break;
177 }
178
179 return index;
180 }
181
pstate_update_perf(struct amd_cpudata * cpudata,u32 min_perf,u32 des_perf,u32 max_perf,bool fast_switch)182 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf,
183 u32 des_perf, u32 max_perf, bool fast_switch)
184 {
185 if (fast_switch)
186 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached));
187 else
188 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
189 READ_ONCE(cpudata->cppc_req_cached));
190 }
191
192 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf);
193
amd_pstate_update_perf(struct amd_cpudata * cpudata,u32 min_perf,u32 des_perf,u32 max_perf,bool fast_switch)194 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata,
195 u32 min_perf, u32 des_perf,
196 u32 max_perf, bool fast_switch)
197 {
198 static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf,
199 max_perf, fast_switch);
200 }
201
amd_pstate_set_epp(struct amd_cpudata * cpudata,u32 epp)202 static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp)
203 {
204 int ret;
205 struct cppc_perf_ctrls perf_ctrls;
206
207 if (boot_cpu_has(X86_FEATURE_CPPC)) {
208 u64 value = READ_ONCE(cpudata->cppc_req_cached);
209
210 value &= ~GENMASK_ULL(31, 24);
211 value |= (u64)epp << 24;
212 WRITE_ONCE(cpudata->cppc_req_cached, value);
213
214 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
215 if (!ret)
216 cpudata->epp_cached = epp;
217 } else {
218 amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U,
219 cpudata->max_limit_perf, false);
220
221 perf_ctrls.energy_perf = epp;
222 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1);
223 if (ret) {
224 pr_debug("failed to set energy perf value (%d)\n", ret);
225 return ret;
226 }
227 cpudata->epp_cached = epp;
228 }
229
230 return ret;
231 }
232
amd_pstate_set_energy_pref_index(struct amd_cpudata * cpudata,int pref_index)233 static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata,
234 int pref_index)
235 {
236 int epp = -EINVAL;
237 int ret;
238
239 if (!pref_index) {
240 pr_debug("EPP pref_index is invalid\n");
241 return -EINVAL;
242 }
243
244 if (epp == -EINVAL)
245 epp = epp_values[pref_index];
246
247 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
248 pr_debug("EPP cannot be set under performance policy\n");
249 return -EBUSY;
250 }
251
252 ret = amd_pstate_set_epp(cpudata, epp);
253
254 return ret;
255 }
256
pstate_enable(bool enable)257 static inline int pstate_enable(bool enable)
258 {
259 int ret, cpu;
260 unsigned long logical_proc_id_mask = 0;
261
262 if (enable == cppc_enabled)
263 return 0;
264
265 for_each_present_cpu(cpu) {
266 unsigned long logical_id = topology_logical_die_id(cpu);
267
268 if (test_bit(logical_id, &logical_proc_id_mask))
269 continue;
270
271 set_bit(logical_id, &logical_proc_id_mask);
272
273 ret = wrmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_ENABLE,
274 enable);
275 if (ret)
276 return ret;
277 }
278
279 cppc_enabled = enable;
280 return 0;
281 }
282
cppc_enable(bool enable)283 static int cppc_enable(bool enable)
284 {
285 int cpu, ret = 0;
286 struct cppc_perf_ctrls perf_ctrls;
287
288 if (enable == cppc_enabled)
289 return 0;
290
291 for_each_present_cpu(cpu) {
292 ret = cppc_set_enable(cpu, enable);
293 if (ret)
294 return ret;
295
296 /* Enable autonomous mode for EPP */
297 if (cppc_state == AMD_PSTATE_ACTIVE) {
298 /* Set desired perf as zero to allow EPP firmware control */
299 perf_ctrls.desired_perf = 0;
300 ret = cppc_set_perf(cpu, &perf_ctrls);
301 if (ret)
302 return ret;
303 }
304 }
305
306 cppc_enabled = enable;
307 return ret;
308 }
309
310 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable);
311
amd_pstate_enable(bool enable)312 static inline int amd_pstate_enable(bool enable)
313 {
314 return static_call(amd_pstate_enable)(enable);
315 }
316
amd_pstate_highest_perf_set(struct amd_cpudata * cpudata)317 static u32 amd_pstate_highest_perf_set(struct amd_cpudata *cpudata)
318 {
319 struct cpuinfo_x86 *c = &cpu_data(0);
320
321 /*
322 * For AMD CPUs with Family ID 19H and Model ID range 0x70 to 0x7f,
323 * the highest performance level is set to 196.
324 * https://bugzilla.kernel.org/show_bug.cgi?id=218759
325 */
326 if (c->x86 == 0x19 && (c->x86_model >= 0x70 && c->x86_model <= 0x7f))
327 return CPPC_HIGHEST_PERF_PERFORMANCE;
328
329 return CPPC_HIGHEST_PERF_DEFAULT;
330 }
331
pstate_init_perf(struct amd_cpudata * cpudata)332 static int pstate_init_perf(struct amd_cpudata *cpudata)
333 {
334 u64 cap1;
335 u32 highest_perf;
336
337 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
338 &cap1);
339 if (ret)
340 return ret;
341
342 /* For platforms that do not support the preferred core feature, the
343 * highest_pef may be configured with 166 or 255, to avoid max frequency
344 * calculated wrongly. we take the AMD_CPPC_HIGHEST_PERF(cap1) value as
345 * the default max perf.
346 */
347 if (cpudata->hw_prefcore)
348 highest_perf = amd_pstate_highest_perf_set(cpudata);
349 else
350 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1);
351
352 WRITE_ONCE(cpudata->highest_perf, highest_perf);
353 WRITE_ONCE(cpudata->max_limit_perf, highest_perf);
354 WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));
355 WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));
356 WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1));
357 WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1));
358 return 0;
359 }
360
cppc_init_perf(struct amd_cpudata * cpudata)361 static int cppc_init_perf(struct amd_cpudata *cpudata)
362 {
363 struct cppc_perf_caps cppc_perf;
364 u32 highest_perf;
365
366 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
367 if (ret)
368 return ret;
369
370 if (cpudata->hw_prefcore)
371 highest_perf = amd_pstate_highest_perf_set(cpudata);
372 else
373 highest_perf = cppc_perf.highest_perf;
374
375 WRITE_ONCE(cpudata->highest_perf, highest_perf);
376 WRITE_ONCE(cpudata->max_limit_perf, highest_perf);
377 WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);
378 WRITE_ONCE(cpudata->lowest_nonlinear_perf,
379 cppc_perf.lowest_nonlinear_perf);
380 WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf);
381 WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf);
382
383 if (cppc_state == AMD_PSTATE_ACTIVE)
384 return 0;
385
386 ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf);
387 if (ret) {
388 pr_warn("failed to get auto_sel, ret: %d\n", ret);
389 return 0;
390 }
391
392 ret = cppc_set_auto_sel(cpudata->cpu,
393 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
394
395 if (ret)
396 pr_warn("failed to set auto_sel, ret: %d\n", ret);
397
398 return ret;
399 }
400
401 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf);
402
amd_pstate_init_perf(struct amd_cpudata * cpudata)403 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)
404 {
405 return static_call(amd_pstate_init_perf)(cpudata);
406 }
407
cppc_update_perf(struct amd_cpudata * cpudata,u32 min_perf,u32 des_perf,u32 max_perf,bool fast_switch)408 static void cppc_update_perf(struct amd_cpudata *cpudata,
409 u32 min_perf, u32 des_perf,
410 u32 max_perf, bool fast_switch)
411 {
412 struct cppc_perf_ctrls perf_ctrls;
413
414 perf_ctrls.max_perf = max_perf;
415 perf_ctrls.min_perf = min_perf;
416 perf_ctrls.desired_perf = des_perf;
417
418 cppc_set_perf(cpudata->cpu, &perf_ctrls);
419 }
420
amd_pstate_sample(struct amd_cpudata * cpudata)421 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)
422 {
423 u64 aperf, mperf, tsc;
424 unsigned long flags;
425
426 local_irq_save(flags);
427 rdmsrl(MSR_IA32_APERF, aperf);
428 rdmsrl(MSR_IA32_MPERF, mperf);
429 tsc = rdtsc();
430
431 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {
432 local_irq_restore(flags);
433 return false;
434 }
435
436 local_irq_restore(flags);
437
438 cpudata->cur.aperf = aperf;
439 cpudata->cur.mperf = mperf;
440 cpudata->cur.tsc = tsc;
441 cpudata->cur.aperf -= cpudata->prev.aperf;
442 cpudata->cur.mperf -= cpudata->prev.mperf;
443 cpudata->cur.tsc -= cpudata->prev.tsc;
444
445 cpudata->prev.aperf = aperf;
446 cpudata->prev.mperf = mperf;
447 cpudata->prev.tsc = tsc;
448
449 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);
450
451 return true;
452 }
453
amd_pstate_update(struct amd_cpudata * cpudata,u32 min_perf,u32 des_perf,u32 max_perf,bool fast_switch,int gov_flags)454 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,
455 u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags)
456 {
457 u64 prev = READ_ONCE(cpudata->cppc_req_cached);
458 u64 value = prev;
459
460 min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf,
461 cpudata->max_limit_perf);
462 max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf,
463 cpudata->max_limit_perf);
464 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
465
466 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) {
467 min_perf = des_perf;
468 des_perf = 0;
469 }
470
471 value &= ~AMD_CPPC_MIN_PERF(~0L);
472 value |= AMD_CPPC_MIN_PERF(min_perf);
473
474 value &= ~AMD_CPPC_DES_PERF(~0L);
475 value |= AMD_CPPC_DES_PERF(des_perf);
476
477 value &= ~AMD_CPPC_MAX_PERF(~0L);
478 value |= AMD_CPPC_MAX_PERF(max_perf);
479
480 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {
481 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,
482 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,
483 cpudata->cpu, (value != prev), fast_switch);
484 }
485
486 if (value == prev)
487 return;
488
489 WRITE_ONCE(cpudata->cppc_req_cached, value);
490
491 amd_pstate_update_perf(cpudata, min_perf, des_perf,
492 max_perf, fast_switch);
493 }
494
amd_pstate_verify(struct cpufreq_policy_data * policy)495 static int amd_pstate_verify(struct cpufreq_policy_data *policy)
496 {
497 cpufreq_verify_within_cpu_limits(policy);
498
499 return 0;
500 }
501
amd_pstate_update_min_max_limit(struct cpufreq_policy * policy)502 static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy)
503 {
504 u32 max_limit_perf, min_limit_perf;
505 struct amd_cpudata *cpudata = policy->driver_data;
506
507 max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq);
508 min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq);
509
510 WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf);
511 WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf);
512 WRITE_ONCE(cpudata->max_limit_freq, policy->max);
513 WRITE_ONCE(cpudata->min_limit_freq, policy->min);
514
515 return 0;
516 }
517
amd_pstate_update_freq(struct cpufreq_policy * policy,unsigned int target_freq,bool fast_switch)518 static int amd_pstate_update_freq(struct cpufreq_policy *policy,
519 unsigned int target_freq, bool fast_switch)
520 {
521 struct cpufreq_freqs freqs;
522 struct amd_cpudata *cpudata = policy->driver_data;
523 unsigned long max_perf, min_perf, des_perf, cap_perf;
524
525 if (!cpudata->max_freq)
526 return -ENODEV;
527
528 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)
529 amd_pstate_update_min_max_limit(policy);
530
531 cap_perf = READ_ONCE(cpudata->highest_perf);
532 min_perf = READ_ONCE(cpudata->lowest_perf);
533 max_perf = cap_perf;
534
535 freqs.old = policy->cur;
536 freqs.new = target_freq;
537
538 des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf,
539 cpudata->max_freq);
540
541 WARN_ON(fast_switch && !policy->fast_switch_enabled);
542 /*
543 * If fast_switch is desired, then there aren't any registered
544 * transition notifiers. See comment for
545 * cpufreq_enable_fast_switch().
546 */
547 if (!fast_switch)
548 cpufreq_freq_transition_begin(policy, &freqs);
549
550 amd_pstate_update(cpudata, min_perf, des_perf,
551 max_perf, fast_switch, policy->governor->flags);
552
553 if (!fast_switch)
554 cpufreq_freq_transition_end(policy, &freqs, false);
555
556 return 0;
557 }
558
amd_pstate_target(struct cpufreq_policy * policy,unsigned int target_freq,unsigned int relation)559 static int amd_pstate_target(struct cpufreq_policy *policy,
560 unsigned int target_freq,
561 unsigned int relation)
562 {
563 return amd_pstate_update_freq(policy, target_freq, false);
564 }
565
amd_pstate_fast_switch(struct cpufreq_policy * policy,unsigned int target_freq)566 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy,
567 unsigned int target_freq)
568 {
569 if (!amd_pstate_update_freq(policy, target_freq, true))
570 return target_freq;
571 return policy->cur;
572 }
573
amd_pstate_adjust_perf(unsigned int cpu,unsigned long _min_perf,unsigned long target_perf,unsigned long capacity)574 static void amd_pstate_adjust_perf(unsigned int cpu,
575 unsigned long _min_perf,
576 unsigned long target_perf,
577 unsigned long capacity)
578 {
579 unsigned long max_perf, min_perf, des_perf,
580 cap_perf, lowest_nonlinear_perf, max_freq;
581 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
582 struct amd_cpudata *cpudata = policy->driver_data;
583 unsigned int target_freq;
584
585 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)
586 amd_pstate_update_min_max_limit(policy);
587
588
589 cap_perf = READ_ONCE(cpudata->highest_perf);
590 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
591 max_freq = READ_ONCE(cpudata->max_freq);
592
593 des_perf = cap_perf;
594 if (target_perf < capacity)
595 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity);
596
597 min_perf = READ_ONCE(cpudata->lowest_perf);
598 if (_min_perf < capacity)
599 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity);
600
601 if (min_perf < lowest_nonlinear_perf)
602 min_perf = lowest_nonlinear_perf;
603
604 max_perf = cap_perf;
605 if (max_perf < min_perf)
606 max_perf = min_perf;
607
608 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
609 target_freq = div_u64(des_perf * max_freq, max_perf);
610 policy->cur = target_freq;
611
612 amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true,
613 policy->governor->flags);
614 cpufreq_cpu_put(policy);
615 }
616
amd_get_min_freq(struct amd_cpudata * cpudata)617 static int amd_get_min_freq(struct amd_cpudata *cpudata)
618 {
619 struct cppc_perf_caps cppc_perf;
620
621 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
622 if (ret)
623 return ret;
624
625 /* Switch to khz */
626 return cppc_perf.lowest_freq * 1000;
627 }
628
amd_get_max_freq(struct amd_cpudata * cpudata)629 static int amd_get_max_freq(struct amd_cpudata *cpudata)
630 {
631 struct cppc_perf_caps cppc_perf;
632 u32 max_perf, max_freq, nominal_freq, nominal_perf;
633 u64 boost_ratio;
634
635 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
636 if (ret)
637 return ret;
638
639 nominal_freq = cppc_perf.nominal_freq;
640 nominal_perf = READ_ONCE(cpudata->nominal_perf);
641 max_perf = READ_ONCE(cpudata->highest_perf);
642
643 boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT,
644 nominal_perf);
645
646 max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT;
647
648 /* Switch to khz */
649 return max_freq * 1000;
650 }
651
amd_get_nominal_freq(struct amd_cpudata * cpudata)652 static int amd_get_nominal_freq(struct amd_cpudata *cpudata)
653 {
654 struct cppc_perf_caps cppc_perf;
655
656 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
657 if (ret)
658 return ret;
659
660 /* Switch to khz */
661 return cppc_perf.nominal_freq * 1000;
662 }
663
amd_get_lowest_nonlinear_freq(struct amd_cpudata * cpudata)664 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata)
665 {
666 struct cppc_perf_caps cppc_perf;
667 u32 lowest_nonlinear_freq, lowest_nonlinear_perf,
668 nominal_freq, nominal_perf;
669 u64 lowest_nonlinear_ratio;
670
671 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
672 if (ret)
673 return ret;
674
675 nominal_freq = cppc_perf.nominal_freq;
676 nominal_perf = READ_ONCE(cpudata->nominal_perf);
677
678 lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf;
679
680 lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT,
681 nominal_perf);
682
683 lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT;
684
685 /* Switch to khz */
686 return lowest_nonlinear_freq * 1000;
687 }
688
amd_pstate_set_boost(struct cpufreq_policy * policy,int state)689 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)
690 {
691 struct amd_cpudata *cpudata = policy->driver_data;
692 int ret;
693
694 if (!cpudata->boost_supported) {
695 pr_err("Boost mode is not supported by this processor or SBIOS\n");
696 return -EINVAL;
697 }
698
699 if (state)
700 policy->cpuinfo.max_freq = cpudata->max_freq;
701 else
702 policy->cpuinfo.max_freq = cpudata->nominal_freq;
703
704 policy->max = policy->cpuinfo.max_freq;
705
706 ret = freq_qos_update_request(&cpudata->req[1],
707 policy->cpuinfo.max_freq);
708 if (ret < 0)
709 return ret;
710
711 return 0;
712 }
713
amd_pstate_boost_init(struct amd_cpudata * cpudata)714 static void amd_pstate_boost_init(struct amd_cpudata *cpudata)
715 {
716 u32 highest_perf, nominal_perf;
717
718 highest_perf = READ_ONCE(cpudata->highest_perf);
719 nominal_perf = READ_ONCE(cpudata->nominal_perf);
720
721 if (highest_perf <= nominal_perf)
722 return;
723
724 cpudata->boost_supported = true;
725 current_pstate_driver->boost_enabled = true;
726 }
727
amd_perf_ctl_reset(unsigned int cpu)728 static void amd_perf_ctl_reset(unsigned int cpu)
729 {
730 wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0);
731 }
732
733 /*
734 * Set amd-pstate preferred core enable can't be done directly from cpufreq callbacks
735 * due to locking, so queue the work for later.
736 */
amd_pstste_sched_prefcore_workfn(struct work_struct * work)737 static void amd_pstste_sched_prefcore_workfn(struct work_struct *work)
738 {
739 sched_set_itmt_support();
740 }
741 static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn);
742
743 /*
744 * Get the highest performance register value.
745 * @cpu: CPU from which to get highest performance.
746 * @highest_perf: Return address.
747 *
748 * Return: 0 for success, -EIO otherwise.
749 */
amd_pstate_get_highest_perf(int cpu,u32 * highest_perf)750 static int amd_pstate_get_highest_perf(int cpu, u32 *highest_perf)
751 {
752 int ret;
753
754 if (boot_cpu_has(X86_FEATURE_CPPC)) {
755 u64 cap1;
756
757 ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1);
758 if (ret)
759 return ret;
760 WRITE_ONCE(*highest_perf, AMD_CPPC_HIGHEST_PERF(cap1));
761 } else {
762 u64 cppc_highest_perf;
763
764 ret = cppc_get_highest_perf(cpu, &cppc_highest_perf);
765 if (ret)
766 return ret;
767 WRITE_ONCE(*highest_perf, cppc_highest_perf);
768 }
769
770 return (ret);
771 }
772
773 #define CPPC_MAX_PERF U8_MAX
774
amd_pstate_init_prefcore(struct amd_cpudata * cpudata)775 static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata)
776 {
777 int ret, prio;
778 u32 highest_perf;
779
780 ret = amd_pstate_get_highest_perf(cpudata->cpu, &highest_perf);
781 if (ret)
782 return;
783
784 cpudata->hw_prefcore = true;
785 /* check if CPPC preferred core feature is enabled*/
786 if (highest_perf < CPPC_MAX_PERF)
787 prio = (int)highest_perf;
788 else {
789 pr_debug("AMD CPPC preferred core is unsupported!\n");
790 cpudata->hw_prefcore = false;
791 return;
792 }
793
794 if (!amd_pstate_prefcore)
795 return;
796
797 /*
798 * The priorities can be set regardless of whether or not
799 * sched_set_itmt_support(true) has been called and it is valid to
800 * update them at any time after it has been called.
801 */
802 sched_set_itmt_core_prio(prio, cpudata->cpu);
803
804 schedule_work(&sched_prefcore_work);
805 }
806
amd_pstate_cpu_init(struct cpufreq_policy * policy)807 static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
808 {
809 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
810 struct device *dev;
811 struct amd_cpudata *cpudata;
812
813 /*
814 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
815 * which is ideal for initialization process.
816 */
817 amd_perf_ctl_reset(policy->cpu);
818 dev = get_cpu_device(policy->cpu);
819 if (!dev)
820 return -ENODEV;
821
822 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
823 if (!cpudata)
824 return -ENOMEM;
825
826 cpudata->cpu = policy->cpu;
827
828 amd_pstate_init_prefcore(cpudata);
829
830 ret = amd_pstate_init_perf(cpudata);
831 if (ret)
832 goto free_cpudata1;
833
834 min_freq = amd_get_min_freq(cpudata);
835 max_freq = amd_get_max_freq(cpudata);
836 nominal_freq = amd_get_nominal_freq(cpudata);
837 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
838
839 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
840 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
841 min_freq, max_freq);
842 ret = -EINVAL;
843 goto free_cpudata1;
844 }
845
846 policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY;
847 policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY;
848
849 policy->min = min_freq;
850 policy->max = max_freq;
851
852 policy->cpuinfo.min_freq = min_freq;
853 policy->cpuinfo.max_freq = max_freq;
854
855 /* It will be updated by governor */
856 policy->cur = policy->cpuinfo.min_freq;
857
858 if (boot_cpu_has(X86_FEATURE_CPPC))
859 policy->fast_switch_possible = true;
860
861 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],
862 FREQ_QOS_MIN, policy->cpuinfo.min_freq);
863 if (ret < 0) {
864 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
865 goto free_cpudata1;
866 }
867
868 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],
869 FREQ_QOS_MAX, policy->cpuinfo.max_freq);
870 if (ret < 0) {
871 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
872 goto free_cpudata2;
873 }
874
875 /* Initial processor data capability frequencies */
876 cpudata->max_freq = max_freq;
877 cpudata->min_freq = min_freq;
878 cpudata->max_limit_freq = max_freq;
879 cpudata->min_limit_freq = min_freq;
880 cpudata->nominal_freq = nominal_freq;
881 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
882
883 policy->driver_data = cpudata;
884
885 amd_pstate_boost_init(cpudata);
886 if (!current_pstate_driver->adjust_perf)
887 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
888
889 return 0;
890
891 free_cpudata2:
892 freq_qos_remove_request(&cpudata->req[0]);
893 free_cpudata1:
894 kfree(cpudata);
895 return ret;
896 }
897
amd_pstate_cpu_exit(struct cpufreq_policy * policy)898 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy)
899 {
900 struct amd_cpudata *cpudata = policy->driver_data;
901
902 freq_qos_remove_request(&cpudata->req[1]);
903 freq_qos_remove_request(&cpudata->req[0]);
904 policy->fast_switch_possible = false;
905 kfree(cpudata);
906
907 return 0;
908 }
909
amd_pstate_cpu_resume(struct cpufreq_policy * policy)910 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy)
911 {
912 int ret;
913
914 ret = amd_pstate_enable(true);
915 if (ret)
916 pr_err("failed to enable amd-pstate during resume, return %d\n", ret);
917
918 return ret;
919 }
920
amd_pstate_cpu_suspend(struct cpufreq_policy * policy)921 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy)
922 {
923 int ret;
924
925 ret = amd_pstate_enable(false);
926 if (ret)
927 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret);
928
929 return ret;
930 }
931
932 /* Sysfs attributes */
933
934 /*
935 * This frequency is to indicate the maximum hardware frequency.
936 * If boost is not active but supported, the frequency will be larger than the
937 * one in cpuinfo.
938 */
show_amd_pstate_max_freq(struct cpufreq_policy * policy,char * buf)939 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,
940 char *buf)
941 {
942 int max_freq;
943 struct amd_cpudata *cpudata = policy->driver_data;
944
945 max_freq = amd_get_max_freq(cpudata);
946 if (max_freq < 0)
947 return max_freq;
948
949 return sysfs_emit(buf, "%u\n", max_freq);
950 }
951
show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy * policy,char * buf)952 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,
953 char *buf)
954 {
955 int freq;
956 struct amd_cpudata *cpudata = policy->driver_data;
957
958 freq = amd_get_lowest_nonlinear_freq(cpudata);
959 if (freq < 0)
960 return freq;
961
962 return sysfs_emit(buf, "%u\n", freq);
963 }
964
965 /*
966 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we
967 * need to expose it to sysfs.
968 */
show_amd_pstate_highest_perf(struct cpufreq_policy * policy,char * buf)969 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,
970 char *buf)
971 {
972 u32 perf;
973 struct amd_cpudata *cpudata = policy->driver_data;
974
975 perf = READ_ONCE(cpudata->highest_perf);
976
977 return sysfs_emit(buf, "%u\n", perf);
978 }
979
show_amd_pstate_hw_prefcore(struct cpufreq_policy * policy,char * buf)980 static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy,
981 char *buf)
982 {
983 bool hw_prefcore;
984 struct amd_cpudata *cpudata = policy->driver_data;
985
986 hw_prefcore = READ_ONCE(cpudata->hw_prefcore);
987
988 return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore));
989 }
990
show_energy_performance_available_preferences(struct cpufreq_policy * policy,char * buf)991 static ssize_t show_energy_performance_available_preferences(
992 struct cpufreq_policy *policy, char *buf)
993 {
994 int i = 0;
995 int offset = 0;
996 struct amd_cpudata *cpudata = policy->driver_data;
997
998 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
999 return sysfs_emit_at(buf, offset, "%s\n",
1000 energy_perf_strings[EPP_INDEX_PERFORMANCE]);
1001
1002 while (energy_perf_strings[i] != NULL)
1003 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]);
1004
1005 offset += sysfs_emit_at(buf, offset, "\n");
1006
1007 return offset;
1008 }
1009
store_energy_performance_preference(struct cpufreq_policy * policy,const char * buf,size_t count)1010 static ssize_t store_energy_performance_preference(
1011 struct cpufreq_policy *policy, const char *buf, size_t count)
1012 {
1013 struct amd_cpudata *cpudata = policy->driver_data;
1014 char str_preference[21];
1015 ssize_t ret;
1016
1017 ret = sscanf(buf, "%20s", str_preference);
1018 if (ret != 1)
1019 return -EINVAL;
1020
1021 ret = match_string(energy_perf_strings, -1, str_preference);
1022 if (ret < 0)
1023 return -EINVAL;
1024
1025 mutex_lock(&amd_pstate_limits_lock);
1026 ret = amd_pstate_set_energy_pref_index(cpudata, ret);
1027 mutex_unlock(&amd_pstate_limits_lock);
1028
1029 return ret ?: count;
1030 }
1031
show_energy_performance_preference(struct cpufreq_policy * policy,char * buf)1032 static ssize_t show_energy_performance_preference(
1033 struct cpufreq_policy *policy, char *buf)
1034 {
1035 struct amd_cpudata *cpudata = policy->driver_data;
1036 int preference;
1037
1038 preference = amd_pstate_get_energy_pref_index(cpudata);
1039 if (preference < 0)
1040 return preference;
1041
1042 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]);
1043 }
1044
amd_pstate_driver_cleanup(void)1045 static void amd_pstate_driver_cleanup(void)
1046 {
1047 amd_pstate_enable(false);
1048 cppc_state = AMD_PSTATE_DISABLE;
1049 current_pstate_driver = NULL;
1050 }
1051
amd_pstate_register_driver(int mode)1052 static int amd_pstate_register_driver(int mode)
1053 {
1054 int ret;
1055
1056 if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED)
1057 current_pstate_driver = &amd_pstate_driver;
1058 else if (mode == AMD_PSTATE_ACTIVE)
1059 current_pstate_driver = &amd_pstate_epp_driver;
1060 else
1061 return -EINVAL;
1062
1063 cppc_state = mode;
1064 ret = cpufreq_register_driver(current_pstate_driver);
1065 if (ret) {
1066 amd_pstate_driver_cleanup();
1067 return ret;
1068 }
1069 return 0;
1070 }
1071
amd_pstate_unregister_driver(int dummy)1072 static int amd_pstate_unregister_driver(int dummy)
1073 {
1074 cpufreq_unregister_driver(current_pstate_driver);
1075 amd_pstate_driver_cleanup();
1076 return 0;
1077 }
1078
amd_pstate_change_mode_without_dvr_change(int mode)1079 static int amd_pstate_change_mode_without_dvr_change(int mode)
1080 {
1081 int cpu = 0;
1082
1083 cppc_state = mode;
1084
1085 if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE)
1086 return 0;
1087
1088 for_each_present_cpu(cpu) {
1089 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
1090 }
1091
1092 return 0;
1093 }
1094
amd_pstate_change_driver_mode(int mode)1095 static int amd_pstate_change_driver_mode(int mode)
1096 {
1097 int ret;
1098
1099 ret = amd_pstate_unregister_driver(0);
1100 if (ret)
1101 return ret;
1102
1103 ret = amd_pstate_register_driver(mode);
1104 if (ret)
1105 return ret;
1106
1107 return 0;
1108 }
1109
1110 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = {
1111 [AMD_PSTATE_DISABLE] = {
1112 [AMD_PSTATE_DISABLE] = NULL,
1113 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver,
1114 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver,
1115 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver,
1116 },
1117 [AMD_PSTATE_PASSIVE] = {
1118 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver,
1119 [AMD_PSTATE_PASSIVE] = NULL,
1120 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode,
1121 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change,
1122 },
1123 [AMD_PSTATE_ACTIVE] = {
1124 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver,
1125 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode,
1126 [AMD_PSTATE_ACTIVE] = NULL,
1127 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode,
1128 },
1129 [AMD_PSTATE_GUIDED] = {
1130 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver,
1131 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change,
1132 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode,
1133 [AMD_PSTATE_GUIDED] = NULL,
1134 },
1135 };
1136
amd_pstate_show_status(char * buf)1137 static ssize_t amd_pstate_show_status(char *buf)
1138 {
1139 if (!current_pstate_driver)
1140 return sysfs_emit(buf, "disable\n");
1141
1142 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]);
1143 }
1144
amd_pstate_update_status(const char * buf,size_t size)1145 static int amd_pstate_update_status(const char *buf, size_t size)
1146 {
1147 int mode_idx;
1148
1149 if (size > strlen("passive") || size < strlen("active"))
1150 return -EINVAL;
1151
1152 mode_idx = get_mode_idx_from_str(buf, size);
1153
1154 if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX)
1155 return -EINVAL;
1156
1157 if (mode_state_machine[cppc_state][mode_idx])
1158 return mode_state_machine[cppc_state][mode_idx](mode_idx);
1159
1160 return 0;
1161 }
1162
status_show(struct device * dev,struct device_attribute * attr,char * buf)1163 static ssize_t status_show(struct device *dev,
1164 struct device_attribute *attr, char *buf)
1165 {
1166 ssize_t ret;
1167
1168 mutex_lock(&amd_pstate_driver_lock);
1169 ret = amd_pstate_show_status(buf);
1170 mutex_unlock(&amd_pstate_driver_lock);
1171
1172 return ret;
1173 }
1174
status_store(struct device * a,struct device_attribute * b,const char * buf,size_t count)1175 static ssize_t status_store(struct device *a, struct device_attribute *b,
1176 const char *buf, size_t count)
1177 {
1178 char *p = memchr(buf, '\n', count);
1179 int ret;
1180
1181 mutex_lock(&amd_pstate_driver_lock);
1182 ret = amd_pstate_update_status(buf, p ? p - buf : count);
1183 mutex_unlock(&amd_pstate_driver_lock);
1184
1185 return ret < 0 ? ret : count;
1186 }
1187
prefcore_show(struct device * dev,struct device_attribute * attr,char * buf)1188 static ssize_t prefcore_show(struct device *dev,
1189 struct device_attribute *attr, char *buf)
1190 {
1191 return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore));
1192 }
1193
1194 cpufreq_freq_attr_ro(amd_pstate_max_freq);
1195 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq);
1196
1197 cpufreq_freq_attr_ro(amd_pstate_highest_perf);
1198 cpufreq_freq_attr_ro(amd_pstate_hw_prefcore);
1199 cpufreq_freq_attr_rw(energy_performance_preference);
1200 cpufreq_freq_attr_ro(energy_performance_available_preferences);
1201 static DEVICE_ATTR_RW(status);
1202 static DEVICE_ATTR_RO(prefcore);
1203
1204 static struct freq_attr *amd_pstate_attr[] = {
1205 &amd_pstate_max_freq,
1206 &amd_pstate_lowest_nonlinear_freq,
1207 &amd_pstate_highest_perf,
1208 &amd_pstate_hw_prefcore,
1209 NULL,
1210 };
1211
1212 static struct freq_attr *amd_pstate_epp_attr[] = {
1213 &amd_pstate_max_freq,
1214 &amd_pstate_lowest_nonlinear_freq,
1215 &amd_pstate_highest_perf,
1216 &amd_pstate_hw_prefcore,
1217 &energy_performance_preference,
1218 &energy_performance_available_preferences,
1219 NULL,
1220 };
1221
1222 static struct attribute *pstate_global_attributes[] = {
1223 &dev_attr_status.attr,
1224 &dev_attr_prefcore.attr,
1225 NULL
1226 };
1227
1228 static const struct attribute_group amd_pstate_global_attr_group = {
1229 .name = "amd_pstate",
1230 .attrs = pstate_global_attributes,
1231 };
1232
amd_pstate_acpi_pm_profile_server(void)1233 static bool amd_pstate_acpi_pm_profile_server(void)
1234 {
1235 switch (acpi_gbl_FADT.preferred_profile) {
1236 case PM_ENTERPRISE_SERVER:
1237 case PM_SOHO_SERVER:
1238 case PM_PERFORMANCE_SERVER:
1239 return true;
1240 }
1241 return false;
1242 }
1243
amd_pstate_acpi_pm_profile_undefined(void)1244 static bool amd_pstate_acpi_pm_profile_undefined(void)
1245 {
1246 if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED)
1247 return true;
1248 if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES)
1249 return true;
1250 return false;
1251 }
1252
amd_pstate_epp_cpu_init(struct cpufreq_policy * policy)1253 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)
1254 {
1255 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
1256 struct amd_cpudata *cpudata;
1257 struct device *dev;
1258 u64 value;
1259
1260 /*
1261 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
1262 * which is ideal for initialization process.
1263 */
1264 amd_perf_ctl_reset(policy->cpu);
1265 dev = get_cpu_device(policy->cpu);
1266 if (!dev)
1267 return -ENODEV;
1268
1269 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
1270 if (!cpudata)
1271 return -ENOMEM;
1272
1273 cpudata->cpu = policy->cpu;
1274 cpudata->epp_policy = 0;
1275
1276 amd_pstate_init_prefcore(cpudata);
1277
1278 ret = amd_pstate_init_perf(cpudata);
1279 if (ret)
1280 goto free_cpudata1;
1281
1282 min_freq = amd_get_min_freq(cpudata);
1283 max_freq = amd_get_max_freq(cpudata);
1284 nominal_freq = amd_get_nominal_freq(cpudata);
1285 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
1286 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
1287 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
1288 min_freq, max_freq);
1289 ret = -EINVAL;
1290 goto free_cpudata1;
1291 }
1292
1293 policy->cpuinfo.min_freq = min_freq;
1294 policy->cpuinfo.max_freq = max_freq;
1295 /* It will be updated by governor */
1296 policy->cur = policy->cpuinfo.min_freq;
1297
1298 /* Initial processor data capability frequencies */
1299 cpudata->max_freq = max_freq;
1300 cpudata->min_freq = min_freq;
1301 cpudata->nominal_freq = nominal_freq;
1302 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
1303
1304 policy->driver_data = cpudata;
1305
1306 cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0);
1307
1308 policy->min = policy->cpuinfo.min_freq;
1309 policy->max = policy->cpuinfo.max_freq;
1310
1311 /*
1312 * Set the policy to provide a valid fallback value in case
1313 * the default cpufreq governor is neither powersave nor performance.
1314 */
1315 if (amd_pstate_acpi_pm_profile_server() ||
1316 amd_pstate_acpi_pm_profile_undefined())
1317 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
1318 else
1319 policy->policy = CPUFREQ_POLICY_POWERSAVE;
1320
1321 if (boot_cpu_has(X86_FEATURE_CPPC)) {
1322 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value);
1323 if (ret)
1324 return ret;
1325 WRITE_ONCE(cpudata->cppc_req_cached, value);
1326
1327 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value);
1328 if (ret)
1329 return ret;
1330 WRITE_ONCE(cpudata->cppc_cap1_cached, value);
1331 }
1332 amd_pstate_boost_init(cpudata);
1333
1334 return 0;
1335
1336 free_cpudata1:
1337 kfree(cpudata);
1338 return ret;
1339 }
1340
amd_pstate_epp_cpu_exit(struct cpufreq_policy * policy)1341 static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy)
1342 {
1343 struct amd_cpudata *cpudata = policy->driver_data;
1344
1345 if (cpudata) {
1346 kfree(cpudata);
1347 policy->driver_data = NULL;
1348 }
1349
1350 pr_debug("CPU %d exiting\n", policy->cpu);
1351 return 0;
1352 }
1353
amd_pstate_epp_update_limit(struct cpufreq_policy * policy)1354 static void amd_pstate_epp_update_limit(struct cpufreq_policy *policy)
1355 {
1356 struct amd_cpudata *cpudata = policy->driver_data;
1357 u32 max_perf, min_perf, min_limit_perf, max_limit_perf;
1358 u64 value;
1359 s16 epp;
1360
1361 max_perf = READ_ONCE(cpudata->highest_perf);
1362 min_perf = READ_ONCE(cpudata->lowest_perf);
1363 max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq);
1364 min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq);
1365
1366 WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf);
1367 WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf);
1368
1369 max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf,
1370 cpudata->max_limit_perf);
1371 min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf,
1372 cpudata->max_limit_perf);
1373 value = READ_ONCE(cpudata->cppc_req_cached);
1374
1375 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1376 min_perf = max_perf;
1377
1378 /* Initial min/max values for CPPC Performance Controls Register */
1379 value &= ~AMD_CPPC_MIN_PERF(~0L);
1380 value |= AMD_CPPC_MIN_PERF(min_perf);
1381
1382 value &= ~AMD_CPPC_MAX_PERF(~0L);
1383 value |= AMD_CPPC_MAX_PERF(max_perf);
1384
1385 /* CPPC EPP feature require to set zero to the desire perf bit */
1386 value &= ~AMD_CPPC_DES_PERF(~0L);
1387 value |= AMD_CPPC_DES_PERF(0);
1388
1389 cpudata->epp_policy = cpudata->policy;
1390
1391 /* Get BIOS pre-defined epp value */
1392 epp = amd_pstate_get_epp(cpudata, value);
1393 if (epp < 0) {
1394 /**
1395 * This return value can only be negative for shared_memory
1396 * systems where EPP register read/write not supported.
1397 */
1398 return;
1399 }
1400
1401 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1402 epp = 0;
1403
1404 /* Set initial EPP value */
1405 if (boot_cpu_has(X86_FEATURE_CPPC)) {
1406 value &= ~GENMASK_ULL(31, 24);
1407 value |= (u64)epp << 24;
1408 }
1409
1410 WRITE_ONCE(cpudata->cppc_req_cached, value);
1411 amd_pstate_set_epp(cpudata, epp);
1412 }
1413
amd_pstate_epp_set_policy(struct cpufreq_policy * policy)1414 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy)
1415 {
1416 struct amd_cpudata *cpudata = policy->driver_data;
1417
1418 if (!policy->cpuinfo.max_freq)
1419 return -ENODEV;
1420
1421 pr_debug("set_policy: cpuinfo.max %u policy->max %u\n",
1422 policy->cpuinfo.max_freq, policy->max);
1423
1424 cpudata->policy = policy->policy;
1425
1426 amd_pstate_epp_update_limit(policy);
1427
1428 return 0;
1429 }
1430
amd_pstate_epp_reenable(struct amd_cpudata * cpudata)1431 static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata)
1432 {
1433 struct cppc_perf_ctrls perf_ctrls;
1434 u64 value, max_perf;
1435 int ret;
1436
1437 ret = amd_pstate_enable(true);
1438 if (ret)
1439 pr_err("failed to enable amd pstate during resume, return %d\n", ret);
1440
1441 value = READ_ONCE(cpudata->cppc_req_cached);
1442 max_perf = READ_ONCE(cpudata->highest_perf);
1443
1444 if (boot_cpu_has(X86_FEATURE_CPPC)) {
1445 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
1446 } else {
1447 perf_ctrls.max_perf = max_perf;
1448 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached);
1449 cppc_set_perf(cpudata->cpu, &perf_ctrls);
1450 }
1451 }
1452
amd_pstate_epp_cpu_online(struct cpufreq_policy * policy)1453 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy)
1454 {
1455 struct amd_cpudata *cpudata = policy->driver_data;
1456
1457 pr_debug("AMD CPU Core %d going online\n", cpudata->cpu);
1458
1459 if (cppc_state == AMD_PSTATE_ACTIVE) {
1460 amd_pstate_epp_reenable(cpudata);
1461 cpudata->suspended = false;
1462 }
1463
1464 return 0;
1465 }
1466
amd_pstate_epp_offline(struct cpufreq_policy * policy)1467 static void amd_pstate_epp_offline(struct cpufreq_policy *policy)
1468 {
1469 struct amd_cpudata *cpudata = policy->driver_data;
1470 struct cppc_perf_ctrls perf_ctrls;
1471 int min_perf;
1472 u64 value;
1473
1474 min_perf = READ_ONCE(cpudata->lowest_perf);
1475 value = READ_ONCE(cpudata->cppc_req_cached);
1476
1477 mutex_lock(&amd_pstate_limits_lock);
1478 if (boot_cpu_has(X86_FEATURE_CPPC)) {
1479 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN;
1480
1481 /* Set max perf same as min perf */
1482 value &= ~AMD_CPPC_MAX_PERF(~0L);
1483 value |= AMD_CPPC_MAX_PERF(min_perf);
1484 value &= ~AMD_CPPC_MIN_PERF(~0L);
1485 value |= AMD_CPPC_MIN_PERF(min_perf);
1486 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
1487 } else {
1488 perf_ctrls.desired_perf = 0;
1489 perf_ctrls.max_perf = min_perf;
1490 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE);
1491 cppc_set_perf(cpudata->cpu, &perf_ctrls);
1492 }
1493 mutex_unlock(&amd_pstate_limits_lock);
1494 }
1495
amd_pstate_epp_cpu_offline(struct cpufreq_policy * policy)1496 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy)
1497 {
1498 struct amd_cpudata *cpudata = policy->driver_data;
1499
1500 pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu);
1501
1502 if (cpudata->suspended)
1503 return 0;
1504
1505 if (cppc_state == AMD_PSTATE_ACTIVE)
1506 amd_pstate_epp_offline(policy);
1507
1508 return 0;
1509 }
1510
amd_pstate_epp_verify_policy(struct cpufreq_policy_data * policy)1511 static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy)
1512 {
1513 cpufreq_verify_within_cpu_limits(policy);
1514 pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min);
1515 return 0;
1516 }
1517
amd_pstate_epp_suspend(struct cpufreq_policy * policy)1518 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy)
1519 {
1520 struct amd_cpudata *cpudata = policy->driver_data;
1521 int ret;
1522
1523 /* avoid suspending when EPP is not enabled */
1524 if (cppc_state != AMD_PSTATE_ACTIVE)
1525 return 0;
1526
1527 /* set this flag to avoid setting core offline*/
1528 cpudata->suspended = true;
1529
1530 /* disable CPPC in lowlevel firmware */
1531 ret = amd_pstate_enable(false);
1532 if (ret)
1533 pr_err("failed to suspend, return %d\n", ret);
1534
1535 return 0;
1536 }
1537
amd_pstate_epp_resume(struct cpufreq_policy * policy)1538 static int amd_pstate_epp_resume(struct cpufreq_policy *policy)
1539 {
1540 struct amd_cpudata *cpudata = policy->driver_data;
1541
1542 if (cpudata->suspended) {
1543 mutex_lock(&amd_pstate_limits_lock);
1544
1545 /* enable amd pstate from suspend state*/
1546 amd_pstate_epp_reenable(cpudata);
1547
1548 mutex_unlock(&amd_pstate_limits_lock);
1549
1550 cpudata->suspended = false;
1551 }
1552
1553 return 0;
1554 }
1555
1556 static struct cpufreq_driver amd_pstate_driver = {
1557 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
1558 .verify = amd_pstate_verify,
1559 .target = amd_pstate_target,
1560 .fast_switch = amd_pstate_fast_switch,
1561 .init = amd_pstate_cpu_init,
1562 .exit = amd_pstate_cpu_exit,
1563 .suspend = amd_pstate_cpu_suspend,
1564 .resume = amd_pstate_cpu_resume,
1565 .set_boost = amd_pstate_set_boost,
1566 .name = "amd-pstate",
1567 .attr = amd_pstate_attr,
1568 };
1569
1570 static struct cpufreq_driver amd_pstate_epp_driver = {
1571 .flags = CPUFREQ_CONST_LOOPS,
1572 .verify = amd_pstate_epp_verify_policy,
1573 .setpolicy = amd_pstate_epp_set_policy,
1574 .init = amd_pstate_epp_cpu_init,
1575 .exit = amd_pstate_epp_cpu_exit,
1576 .offline = amd_pstate_epp_cpu_offline,
1577 .online = amd_pstate_epp_cpu_online,
1578 .suspend = amd_pstate_epp_suspend,
1579 .resume = amd_pstate_epp_resume,
1580 .name = "amd-pstate-epp",
1581 .attr = amd_pstate_epp_attr,
1582 };
1583
amd_pstate_set_driver(int mode_idx)1584 static int __init amd_pstate_set_driver(int mode_idx)
1585 {
1586 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) {
1587 cppc_state = mode_idx;
1588 if (cppc_state == AMD_PSTATE_DISABLE)
1589 pr_info("driver is explicitly disabled\n");
1590
1591 if (cppc_state == AMD_PSTATE_ACTIVE)
1592 current_pstate_driver = &amd_pstate_epp_driver;
1593
1594 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED)
1595 current_pstate_driver = &amd_pstate_driver;
1596
1597 return 0;
1598 }
1599
1600 return -EINVAL;
1601 }
1602
amd_pstate_init(void)1603 static int __init amd_pstate_init(void)
1604 {
1605 struct device *dev_root;
1606 int ret;
1607
1608 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
1609 return -ENODEV;
1610
1611 if (!acpi_cpc_valid()) {
1612 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
1613 return -ENODEV;
1614 }
1615
1616 /* don't keep reloading if cpufreq_driver exists */
1617 if (cpufreq_get_current_driver())
1618 return -EEXIST;
1619
1620 switch (cppc_state) {
1621 case AMD_PSTATE_UNDEFINED:
1622 /* Disable on the following configs by default:
1623 * 1. Undefined platforms
1624 * 2. Server platforms
1625 * 3. Shared memory designs
1626 */
1627 if (amd_pstate_acpi_pm_profile_undefined() ||
1628 amd_pstate_acpi_pm_profile_server() ||
1629 !boot_cpu_has(X86_FEATURE_CPPC)) {
1630 pr_info("driver load is disabled, boot with specific mode to enable this\n");
1631 return -ENODEV;
1632 }
1633 ret = amd_pstate_set_driver(CONFIG_X86_AMD_PSTATE_DEFAULT_MODE);
1634 if (ret)
1635 return ret;
1636 break;
1637 case AMD_PSTATE_DISABLE:
1638 return -ENODEV;
1639 case AMD_PSTATE_PASSIVE:
1640 case AMD_PSTATE_ACTIVE:
1641 case AMD_PSTATE_GUIDED:
1642 break;
1643 default:
1644 return -EINVAL;
1645 }
1646
1647 /* capability check */
1648 if (boot_cpu_has(X86_FEATURE_CPPC)) {
1649 pr_debug("AMD CPPC MSR based functionality is supported\n");
1650 if (cppc_state != AMD_PSTATE_ACTIVE)
1651 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
1652 } else {
1653 pr_debug("AMD CPPC shared memory based functionality is supported\n");
1654 static_call_update(amd_pstate_enable, cppc_enable);
1655 static_call_update(amd_pstate_init_perf, cppc_init_perf);
1656 static_call_update(amd_pstate_update_perf, cppc_update_perf);
1657 }
1658
1659 /* enable amd pstate feature */
1660 ret = amd_pstate_enable(true);
1661 if (ret) {
1662 pr_err("failed to enable with return %d\n", ret);
1663 return ret;
1664 }
1665
1666 ret = cpufreq_register_driver(current_pstate_driver);
1667 if (ret)
1668 pr_err("failed to register with return %d\n", ret);
1669
1670 dev_root = bus_get_dev_root(&cpu_subsys);
1671 if (dev_root) {
1672 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group);
1673 put_device(dev_root);
1674 if (ret) {
1675 pr_err("sysfs attribute export failed with error %d.\n", ret);
1676 goto global_attr_free;
1677 }
1678 }
1679
1680 return ret;
1681
1682 global_attr_free:
1683 cpufreq_unregister_driver(current_pstate_driver);
1684 return ret;
1685 }
1686 device_initcall(amd_pstate_init);
1687
amd_pstate_param(char * str)1688 static int __init amd_pstate_param(char *str)
1689 {
1690 size_t size;
1691 int mode_idx;
1692
1693 if (!str)
1694 return -EINVAL;
1695
1696 size = strlen(str);
1697 mode_idx = get_mode_idx_from_str(str, size);
1698
1699 return amd_pstate_set_driver(mode_idx);
1700 }
1701
amd_prefcore_param(char * str)1702 static int __init amd_prefcore_param(char *str)
1703 {
1704 if (!strcmp(str, "disable"))
1705 amd_pstate_prefcore = false;
1706
1707 return 0;
1708 }
1709
1710 early_param("amd_pstate", amd_pstate_param);
1711 early_param("amd_prefcore", amd_prefcore_param);
1712
1713 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>");
1714 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");
1715