1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/kernel.h> 26 #include <linux/module.h> 27 #include <linux/init.h> 28 #include <linux/smp.h> 29 #include <linux/sched.h> 30 #include <linux/cpufreq.h> 31 #include <linux/compiler.h> 32 #include <linux/dmi.h> 33 #include <linux/slab.h> 34 #include <linux/acpi.h> 35 #include <linux/io.h> 36 #include <linux/delay.h> 37 #include <linux/uaccess.h> 38 #include <linux/static_call.h> 39 #include <linux/amd-pstate.h> 40 41 #include <acpi/processor.h> 42 #include <acpi/cppc_acpi.h> 43 44 #include <asm/msr.h> 45 #include <asm/processor.h> 46 #include <asm/cpufeature.h> 47 #include <asm/cpu_device_id.h> 48 #include "amd-pstate-trace.h" 49 50 #define AMD_PSTATE_TRANSITION_LATENCY 20000 51 #define AMD_PSTATE_TRANSITION_DELAY 1000 52 53 /* 54 * TODO: We need more time to fine tune processors with shared memory solution 55 * with community together. 56 * 57 * There are some performance drops on the CPU benchmarks which reports from 58 * Suse. We are co-working with them to fine tune the shared memory solution. So 59 * we disable it by default to go acpi-cpufreq on these processors and add a 60 * module parameter to be able to enable it manually for debugging. 61 */ 62 static struct cpufreq_driver *current_pstate_driver; 63 static struct cpufreq_driver amd_pstate_driver; 64 static struct cpufreq_driver amd_pstate_epp_driver; 65 static int cppc_state = AMD_PSTATE_DISABLE; 66 67 /* 68 * AMD Energy Preference Performance (EPP) 69 * The EPP is used in the CCLK DPM controller to drive 70 * the frequency that a core is going to operate during 71 * short periods of activity. EPP values will be utilized for 72 * different OS profiles (balanced, performance, power savings) 73 * display strings corresponding to EPP index in the 74 * energy_perf_strings[] 75 * index String 76 *------------------------------------- 77 * 0 default 78 * 1 performance 79 * 2 balance_performance 80 * 3 balance_power 81 * 4 power 82 */ 83 enum energy_perf_value_index { 84 EPP_INDEX_DEFAULT = 0, 85 EPP_INDEX_PERFORMANCE, 86 EPP_INDEX_BALANCE_PERFORMANCE, 87 EPP_INDEX_BALANCE_POWERSAVE, 88 EPP_INDEX_POWERSAVE, 89 }; 90 91 static const char * const energy_perf_strings[] = { 92 [EPP_INDEX_DEFAULT] = "default", 93 [EPP_INDEX_PERFORMANCE] = "performance", 94 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 95 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 96 [EPP_INDEX_POWERSAVE] = "power", 97 NULL 98 }; 99 100 static unsigned int epp_values[] = { 101 [EPP_INDEX_DEFAULT] = 0, 102 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 103 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 104 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 105 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 106 }; 107 108 typedef int (*cppc_mode_transition_fn)(int); 109 110 static inline int get_mode_idx_from_str(const char *str, size_t size) 111 { 112 int i; 113 114 for (i=0; i < AMD_PSTATE_MAX; i++) { 115 if (!strncmp(str, amd_pstate_mode_string[i], size)) 116 return i; 117 } 118 return -EINVAL; 119 } 120 121 static DEFINE_MUTEX(amd_pstate_limits_lock); 122 static DEFINE_MUTEX(amd_pstate_driver_lock); 123 124 static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) 125 { 126 u64 epp; 127 int ret; 128 129 if (boot_cpu_has(X86_FEATURE_CPPC)) { 130 if (!cppc_req_cached) { 131 epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 132 &cppc_req_cached); 133 if (epp) 134 return epp; 135 } 136 epp = (cppc_req_cached >> 24) & 0xFF; 137 } else { 138 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 139 if (ret < 0) { 140 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 141 return -EIO; 142 } 143 } 144 145 return (s16)(epp & 0xff); 146 } 147 148 static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) 149 { 150 s16 epp; 151 int index = -EINVAL; 152 153 epp = amd_pstate_get_epp(cpudata, 0); 154 if (epp < 0) 155 return epp; 156 157 switch (epp) { 158 case AMD_CPPC_EPP_PERFORMANCE: 159 index = EPP_INDEX_PERFORMANCE; 160 break; 161 case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 162 index = EPP_INDEX_BALANCE_PERFORMANCE; 163 break; 164 case AMD_CPPC_EPP_BALANCE_POWERSAVE: 165 index = EPP_INDEX_BALANCE_POWERSAVE; 166 break; 167 case AMD_CPPC_EPP_POWERSAVE: 168 index = EPP_INDEX_POWERSAVE; 169 break; 170 default: 171 break; 172 } 173 174 return index; 175 } 176 177 static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) 178 { 179 int ret; 180 struct cppc_perf_ctrls perf_ctrls; 181 182 if (boot_cpu_has(X86_FEATURE_CPPC)) { 183 u64 value = READ_ONCE(cpudata->cppc_req_cached); 184 185 value &= ~GENMASK_ULL(31, 24); 186 value |= (u64)epp << 24; 187 WRITE_ONCE(cpudata->cppc_req_cached, value); 188 189 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 190 if (!ret) 191 cpudata->epp_cached = epp; 192 } else { 193 perf_ctrls.energy_perf = epp; 194 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 195 if (ret) { 196 pr_debug("failed to set energy perf value (%d)\n", ret); 197 return ret; 198 } 199 cpudata->epp_cached = epp; 200 } 201 202 return ret; 203 } 204 205 static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, 206 int pref_index) 207 { 208 int epp = -EINVAL; 209 int ret; 210 211 if (!pref_index) { 212 pr_debug("EPP pref_index is invalid\n"); 213 return -EINVAL; 214 } 215 216 if (epp == -EINVAL) 217 epp = epp_values[pref_index]; 218 219 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 220 pr_debug("EPP cannot be set under performance policy\n"); 221 return -EBUSY; 222 } 223 224 ret = amd_pstate_set_epp(cpudata, epp); 225 226 return ret; 227 } 228 229 static inline int pstate_enable(bool enable) 230 { 231 return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); 232 } 233 234 static int cppc_enable(bool enable) 235 { 236 int cpu, ret = 0; 237 struct cppc_perf_ctrls perf_ctrls; 238 239 for_each_present_cpu(cpu) { 240 ret = cppc_set_enable(cpu, enable); 241 if (ret) 242 return ret; 243 244 /* Enable autonomous mode for EPP */ 245 if (cppc_state == AMD_PSTATE_ACTIVE) { 246 /* Set desired perf as zero to allow EPP firmware control */ 247 perf_ctrls.desired_perf = 0; 248 ret = cppc_set_perf(cpu, &perf_ctrls); 249 if (ret) 250 return ret; 251 } 252 } 253 254 return ret; 255 } 256 257 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); 258 259 static inline int amd_pstate_enable(bool enable) 260 { 261 return static_call(amd_pstate_enable)(enable); 262 } 263 264 static int pstate_init_perf(struct amd_cpudata *cpudata) 265 { 266 u64 cap1; 267 u32 highest_perf; 268 269 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 270 &cap1); 271 if (ret) 272 return ret; 273 274 /* 275 * TODO: Introduce AMD specific power feature. 276 * 277 * CPPC entry doesn't indicate the highest performance in some ASICs. 278 */ 279 highest_perf = amd_get_highest_perf(); 280 if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) 281 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); 282 283 WRITE_ONCE(cpudata->highest_perf, highest_perf); 284 285 WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); 286 WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); 287 WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); 288 289 return 0; 290 } 291 292 static int cppc_init_perf(struct amd_cpudata *cpudata) 293 { 294 struct cppc_perf_caps cppc_perf; 295 u32 highest_perf; 296 297 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 298 if (ret) 299 return ret; 300 301 highest_perf = amd_get_highest_perf(); 302 if (highest_perf > cppc_perf.highest_perf) 303 highest_perf = cppc_perf.highest_perf; 304 305 WRITE_ONCE(cpudata->highest_perf, highest_perf); 306 307 WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); 308 WRITE_ONCE(cpudata->lowest_nonlinear_perf, 309 cppc_perf.lowest_nonlinear_perf); 310 WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); 311 312 if (cppc_state == AMD_PSTATE_ACTIVE) 313 return 0; 314 315 ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf); 316 if (ret) { 317 pr_warn("failed to get auto_sel, ret: %d\n", ret); 318 return 0; 319 } 320 321 ret = cppc_set_auto_sel(cpudata->cpu, 322 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 323 324 if (ret) 325 pr_warn("failed to set auto_sel, ret: %d\n", ret); 326 327 return ret; 328 } 329 330 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); 331 332 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 333 { 334 return static_call(amd_pstate_init_perf)(cpudata); 335 } 336 337 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, 338 u32 des_perf, u32 max_perf, bool fast_switch) 339 { 340 if (fast_switch) 341 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); 342 else 343 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 344 READ_ONCE(cpudata->cppc_req_cached)); 345 } 346 347 static void cppc_update_perf(struct amd_cpudata *cpudata, 348 u32 min_perf, u32 des_perf, 349 u32 max_perf, bool fast_switch) 350 { 351 struct cppc_perf_ctrls perf_ctrls; 352 353 perf_ctrls.max_perf = max_perf; 354 perf_ctrls.min_perf = min_perf; 355 perf_ctrls.desired_perf = des_perf; 356 357 cppc_set_perf(cpudata->cpu, &perf_ctrls); 358 } 359 360 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); 361 362 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, 363 u32 min_perf, u32 des_perf, 364 u32 max_perf, bool fast_switch) 365 { 366 static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, 367 max_perf, fast_switch); 368 } 369 370 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 371 { 372 u64 aperf, mperf, tsc; 373 unsigned long flags; 374 375 local_irq_save(flags); 376 rdmsrl(MSR_IA32_APERF, aperf); 377 rdmsrl(MSR_IA32_MPERF, mperf); 378 tsc = rdtsc(); 379 380 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 381 local_irq_restore(flags); 382 return false; 383 } 384 385 local_irq_restore(flags); 386 387 cpudata->cur.aperf = aperf; 388 cpudata->cur.mperf = mperf; 389 cpudata->cur.tsc = tsc; 390 cpudata->cur.aperf -= cpudata->prev.aperf; 391 cpudata->cur.mperf -= cpudata->prev.mperf; 392 cpudata->cur.tsc -= cpudata->prev.tsc; 393 394 cpudata->prev.aperf = aperf; 395 cpudata->prev.mperf = mperf; 396 cpudata->prev.tsc = tsc; 397 398 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 399 400 return true; 401 } 402 403 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, 404 u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags) 405 { 406 u64 prev = READ_ONCE(cpudata->cppc_req_cached); 407 u64 value = prev; 408 409 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 410 411 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 412 min_perf = des_perf; 413 des_perf = 0; 414 } 415 416 value &= ~AMD_CPPC_MIN_PERF(~0L); 417 value |= AMD_CPPC_MIN_PERF(min_perf); 418 419 value &= ~AMD_CPPC_DES_PERF(~0L); 420 value |= AMD_CPPC_DES_PERF(des_perf); 421 422 value &= ~AMD_CPPC_MAX_PERF(~0L); 423 value |= AMD_CPPC_MAX_PERF(max_perf); 424 425 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 426 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 427 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 428 cpudata->cpu, (value != prev), fast_switch); 429 } 430 431 if (value == prev) 432 return; 433 434 WRITE_ONCE(cpudata->cppc_req_cached, value); 435 436 amd_pstate_update_perf(cpudata, min_perf, des_perf, 437 max_perf, fast_switch); 438 } 439 440 static int amd_pstate_verify(struct cpufreq_policy_data *policy) 441 { 442 cpufreq_verify_within_cpu_limits(policy); 443 444 return 0; 445 } 446 447 static int amd_pstate_update_freq(struct cpufreq_policy *policy, 448 unsigned int target_freq, bool fast_switch) 449 { 450 struct cpufreq_freqs freqs; 451 struct amd_cpudata *cpudata = policy->driver_data; 452 unsigned long max_perf, min_perf, des_perf, cap_perf; 453 454 if (!cpudata->max_freq) 455 return -ENODEV; 456 457 cap_perf = READ_ONCE(cpudata->highest_perf); 458 min_perf = READ_ONCE(cpudata->lowest_perf); 459 max_perf = cap_perf; 460 461 freqs.old = policy->cur; 462 freqs.new = target_freq; 463 464 des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, 465 cpudata->max_freq); 466 467 WARN_ON(fast_switch && !policy->fast_switch_enabled); 468 /* 469 * If fast_switch is desired, then there aren't any registered 470 * transition notifiers. See comment for 471 * cpufreq_enable_fast_switch(). 472 */ 473 if (!fast_switch) 474 cpufreq_freq_transition_begin(policy, &freqs); 475 476 amd_pstate_update(cpudata, min_perf, des_perf, 477 max_perf, fast_switch, policy->governor->flags); 478 479 if (!fast_switch) 480 cpufreq_freq_transition_end(policy, &freqs, false); 481 482 return 0; 483 } 484 485 static int amd_pstate_target(struct cpufreq_policy *policy, 486 unsigned int target_freq, 487 unsigned int relation) 488 { 489 return amd_pstate_update_freq(policy, target_freq, false); 490 } 491 492 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, 493 unsigned int target_freq) 494 { 495 return amd_pstate_update_freq(policy, target_freq, true); 496 } 497 498 static void amd_pstate_adjust_perf(unsigned int cpu, 499 unsigned long _min_perf, 500 unsigned long target_perf, 501 unsigned long capacity) 502 { 503 unsigned long max_perf, min_perf, des_perf, 504 cap_perf, lowest_nonlinear_perf, max_freq; 505 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 506 struct amd_cpudata *cpudata = policy->driver_data; 507 unsigned int target_freq; 508 509 cap_perf = READ_ONCE(cpudata->highest_perf); 510 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); 511 max_freq = READ_ONCE(cpudata->max_freq); 512 513 des_perf = cap_perf; 514 if (target_perf < capacity) 515 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 516 517 min_perf = READ_ONCE(cpudata->highest_perf); 518 if (_min_perf < capacity) 519 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 520 521 if (min_perf < lowest_nonlinear_perf) 522 min_perf = lowest_nonlinear_perf; 523 524 max_perf = cap_perf; 525 if (max_perf < min_perf) 526 max_perf = min_perf; 527 528 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 529 target_freq = div_u64(des_perf * max_freq, max_perf); 530 policy->cur = target_freq; 531 532 amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, 533 policy->governor->flags); 534 cpufreq_cpu_put(policy); 535 } 536 537 static int amd_get_min_freq(struct amd_cpudata *cpudata) 538 { 539 struct cppc_perf_caps cppc_perf; 540 541 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 542 if (ret) 543 return ret; 544 545 /* Switch to khz */ 546 return cppc_perf.lowest_freq * 1000; 547 } 548 549 static int amd_get_max_freq(struct amd_cpudata *cpudata) 550 { 551 struct cppc_perf_caps cppc_perf; 552 u32 max_perf, max_freq, nominal_freq, nominal_perf; 553 u64 boost_ratio; 554 555 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 556 if (ret) 557 return ret; 558 559 nominal_freq = cppc_perf.nominal_freq; 560 nominal_perf = READ_ONCE(cpudata->nominal_perf); 561 max_perf = READ_ONCE(cpudata->highest_perf); 562 563 boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT, 564 nominal_perf); 565 566 max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT; 567 568 /* Switch to khz */ 569 return max_freq * 1000; 570 } 571 572 static int amd_get_nominal_freq(struct amd_cpudata *cpudata) 573 { 574 struct cppc_perf_caps cppc_perf; 575 576 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 577 if (ret) 578 return ret; 579 580 /* Switch to khz */ 581 return cppc_perf.nominal_freq * 1000; 582 } 583 584 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) 585 { 586 struct cppc_perf_caps cppc_perf; 587 u32 lowest_nonlinear_freq, lowest_nonlinear_perf, 588 nominal_freq, nominal_perf; 589 u64 lowest_nonlinear_ratio; 590 591 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 592 if (ret) 593 return ret; 594 595 nominal_freq = cppc_perf.nominal_freq; 596 nominal_perf = READ_ONCE(cpudata->nominal_perf); 597 598 lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 599 600 lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, 601 nominal_perf); 602 603 lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT; 604 605 /* Switch to khz */ 606 return lowest_nonlinear_freq * 1000; 607 } 608 609 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 610 { 611 struct amd_cpudata *cpudata = policy->driver_data; 612 int ret; 613 614 if (!cpudata->boost_supported) { 615 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 616 return -EINVAL; 617 } 618 619 if (state) 620 policy->cpuinfo.max_freq = cpudata->max_freq; 621 else 622 policy->cpuinfo.max_freq = cpudata->nominal_freq; 623 624 policy->max = policy->cpuinfo.max_freq; 625 626 ret = freq_qos_update_request(&cpudata->req[1], 627 policy->cpuinfo.max_freq); 628 if (ret < 0) 629 return ret; 630 631 return 0; 632 } 633 634 static void amd_pstate_boost_init(struct amd_cpudata *cpudata) 635 { 636 u32 highest_perf, nominal_perf; 637 638 highest_perf = READ_ONCE(cpudata->highest_perf); 639 nominal_perf = READ_ONCE(cpudata->nominal_perf); 640 641 if (highest_perf <= nominal_perf) 642 return; 643 644 cpudata->boost_supported = true; 645 current_pstate_driver->boost_enabled = true; 646 } 647 648 static void amd_perf_ctl_reset(unsigned int cpu) 649 { 650 wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 651 } 652 653 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 654 { 655 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 656 struct device *dev; 657 struct amd_cpudata *cpudata; 658 659 /* 660 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 661 * which is ideal for initialization process. 662 */ 663 amd_perf_ctl_reset(policy->cpu); 664 dev = get_cpu_device(policy->cpu); 665 if (!dev) 666 return -ENODEV; 667 668 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 669 if (!cpudata) 670 return -ENOMEM; 671 672 cpudata->cpu = policy->cpu; 673 674 ret = amd_pstate_init_perf(cpudata); 675 if (ret) 676 goto free_cpudata1; 677 678 min_freq = amd_get_min_freq(cpudata); 679 max_freq = amd_get_max_freq(cpudata); 680 nominal_freq = amd_get_nominal_freq(cpudata); 681 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 682 683 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 684 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 685 min_freq, max_freq); 686 ret = -EINVAL; 687 goto free_cpudata1; 688 } 689 690 policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; 691 policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; 692 693 policy->min = min_freq; 694 policy->max = max_freq; 695 696 policy->cpuinfo.min_freq = min_freq; 697 policy->cpuinfo.max_freq = max_freq; 698 699 /* It will be updated by governor */ 700 policy->cur = policy->cpuinfo.min_freq; 701 702 if (boot_cpu_has(X86_FEATURE_CPPC)) 703 policy->fast_switch_possible = true; 704 705 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 706 FREQ_QOS_MIN, policy->cpuinfo.min_freq); 707 if (ret < 0) { 708 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 709 goto free_cpudata1; 710 } 711 712 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 713 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 714 if (ret < 0) { 715 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 716 goto free_cpudata2; 717 } 718 719 /* Initial processor data capability frequencies */ 720 cpudata->max_freq = max_freq; 721 cpudata->min_freq = min_freq; 722 cpudata->nominal_freq = nominal_freq; 723 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 724 725 policy->driver_data = cpudata; 726 727 amd_pstate_boost_init(cpudata); 728 if (!current_pstate_driver->adjust_perf) 729 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 730 731 return 0; 732 733 free_cpudata2: 734 freq_qos_remove_request(&cpudata->req[0]); 735 free_cpudata1: 736 kfree(cpudata); 737 return ret; 738 } 739 740 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) 741 { 742 struct amd_cpudata *cpudata = policy->driver_data; 743 744 freq_qos_remove_request(&cpudata->req[1]); 745 freq_qos_remove_request(&cpudata->req[0]); 746 policy->fast_switch_possible = false; 747 kfree(cpudata); 748 749 return 0; 750 } 751 752 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) 753 { 754 int ret; 755 756 ret = amd_pstate_enable(true); 757 if (ret) 758 pr_err("failed to enable amd-pstate during resume, return %d\n", ret); 759 760 return ret; 761 } 762 763 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) 764 { 765 int ret; 766 767 ret = amd_pstate_enable(false); 768 if (ret) 769 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); 770 771 return ret; 772 } 773 774 /* Sysfs attributes */ 775 776 /* 777 * This frequency is to indicate the maximum hardware frequency. 778 * If boost is not active but supported, the frequency will be larger than the 779 * one in cpuinfo. 780 */ 781 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 782 char *buf) 783 { 784 int max_freq; 785 struct amd_cpudata *cpudata = policy->driver_data; 786 787 max_freq = amd_get_max_freq(cpudata); 788 if (max_freq < 0) 789 return max_freq; 790 791 return sysfs_emit(buf, "%u\n", max_freq); 792 } 793 794 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 795 char *buf) 796 { 797 int freq; 798 struct amd_cpudata *cpudata = policy->driver_data; 799 800 freq = amd_get_lowest_nonlinear_freq(cpudata); 801 if (freq < 0) 802 return freq; 803 804 return sysfs_emit(buf, "%u\n", freq); 805 } 806 807 /* 808 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 809 * need to expose it to sysfs. 810 */ 811 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 812 char *buf) 813 { 814 u32 perf; 815 struct amd_cpudata *cpudata = policy->driver_data; 816 817 perf = READ_ONCE(cpudata->highest_perf); 818 819 return sysfs_emit(buf, "%u\n", perf); 820 } 821 822 static ssize_t show_energy_performance_available_preferences( 823 struct cpufreq_policy *policy, char *buf) 824 { 825 int i = 0; 826 int offset = 0; 827 828 while (energy_perf_strings[i] != NULL) 829 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); 830 831 sysfs_emit_at(buf, offset, "\n"); 832 833 return offset; 834 } 835 836 static ssize_t store_energy_performance_preference( 837 struct cpufreq_policy *policy, const char *buf, size_t count) 838 { 839 struct amd_cpudata *cpudata = policy->driver_data; 840 char str_preference[21]; 841 ssize_t ret; 842 843 ret = sscanf(buf, "%20s", str_preference); 844 if (ret != 1) 845 return -EINVAL; 846 847 ret = match_string(energy_perf_strings, -1, str_preference); 848 if (ret < 0) 849 return -EINVAL; 850 851 mutex_lock(&amd_pstate_limits_lock); 852 ret = amd_pstate_set_energy_pref_index(cpudata, ret); 853 mutex_unlock(&amd_pstate_limits_lock); 854 855 return ret ?: count; 856 } 857 858 static ssize_t show_energy_performance_preference( 859 struct cpufreq_policy *policy, char *buf) 860 { 861 struct amd_cpudata *cpudata = policy->driver_data; 862 int preference; 863 864 preference = amd_pstate_get_energy_pref_index(cpudata); 865 if (preference < 0) 866 return preference; 867 868 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 869 } 870 871 static void amd_pstate_driver_cleanup(void) 872 { 873 amd_pstate_enable(false); 874 cppc_state = AMD_PSTATE_DISABLE; 875 current_pstate_driver = NULL; 876 } 877 878 static int amd_pstate_register_driver(int mode) 879 { 880 int ret; 881 882 if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED) 883 current_pstate_driver = &amd_pstate_driver; 884 else if (mode == AMD_PSTATE_ACTIVE) 885 current_pstate_driver = &amd_pstate_epp_driver; 886 else 887 return -EINVAL; 888 889 cppc_state = mode; 890 ret = cpufreq_register_driver(current_pstate_driver); 891 if (ret) { 892 amd_pstate_driver_cleanup(); 893 return ret; 894 } 895 return 0; 896 } 897 898 static int amd_pstate_unregister_driver(int dummy) 899 { 900 cpufreq_unregister_driver(current_pstate_driver); 901 amd_pstate_driver_cleanup(); 902 return 0; 903 } 904 905 static int amd_pstate_change_mode_without_dvr_change(int mode) 906 { 907 int cpu = 0; 908 909 cppc_state = mode; 910 911 if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 912 return 0; 913 914 for_each_present_cpu(cpu) { 915 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 916 } 917 918 return 0; 919 } 920 921 static int amd_pstate_change_driver_mode(int mode) 922 { 923 int ret; 924 925 ret = amd_pstate_unregister_driver(0); 926 if (ret) 927 return ret; 928 929 ret = amd_pstate_register_driver(mode); 930 if (ret) 931 return ret; 932 933 return 0; 934 } 935 936 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 937 [AMD_PSTATE_DISABLE] = { 938 [AMD_PSTATE_DISABLE] = NULL, 939 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 940 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 941 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 942 }, 943 [AMD_PSTATE_PASSIVE] = { 944 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 945 [AMD_PSTATE_PASSIVE] = NULL, 946 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 947 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 948 }, 949 [AMD_PSTATE_ACTIVE] = { 950 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 951 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 952 [AMD_PSTATE_ACTIVE] = NULL, 953 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 954 }, 955 [AMD_PSTATE_GUIDED] = { 956 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 957 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 958 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 959 [AMD_PSTATE_GUIDED] = NULL, 960 }, 961 }; 962 963 static ssize_t amd_pstate_show_status(char *buf) 964 { 965 if (!current_pstate_driver) 966 return sysfs_emit(buf, "disable\n"); 967 968 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 969 } 970 971 static int amd_pstate_update_status(const char *buf, size_t size) 972 { 973 int mode_idx; 974 975 if (size > strlen("passive") || size < strlen("active")) 976 return -EINVAL; 977 978 mode_idx = get_mode_idx_from_str(buf, size); 979 980 if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX) 981 return -EINVAL; 982 983 if (mode_state_machine[cppc_state][mode_idx]) 984 return mode_state_machine[cppc_state][mode_idx](mode_idx); 985 986 return 0; 987 } 988 989 static ssize_t show_status(struct kobject *kobj, 990 struct kobj_attribute *attr, char *buf) 991 { 992 ssize_t ret; 993 994 mutex_lock(&amd_pstate_driver_lock); 995 ret = amd_pstate_show_status(buf); 996 mutex_unlock(&amd_pstate_driver_lock); 997 998 return ret; 999 } 1000 1001 static ssize_t store_status(struct kobject *a, struct kobj_attribute *b, 1002 const char *buf, size_t count) 1003 { 1004 char *p = memchr(buf, '\n', count); 1005 int ret; 1006 1007 mutex_lock(&amd_pstate_driver_lock); 1008 ret = amd_pstate_update_status(buf, p ? p - buf : count); 1009 mutex_unlock(&amd_pstate_driver_lock); 1010 1011 return ret < 0 ? ret : count; 1012 } 1013 1014 cpufreq_freq_attr_ro(amd_pstate_max_freq); 1015 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 1016 1017 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 1018 cpufreq_freq_attr_rw(energy_performance_preference); 1019 cpufreq_freq_attr_ro(energy_performance_available_preferences); 1020 define_one_global_rw(status); 1021 1022 static struct freq_attr *amd_pstate_attr[] = { 1023 &amd_pstate_max_freq, 1024 &amd_pstate_lowest_nonlinear_freq, 1025 &amd_pstate_highest_perf, 1026 NULL, 1027 }; 1028 1029 static struct freq_attr *amd_pstate_epp_attr[] = { 1030 &amd_pstate_max_freq, 1031 &amd_pstate_lowest_nonlinear_freq, 1032 &amd_pstate_highest_perf, 1033 &energy_performance_preference, 1034 &energy_performance_available_preferences, 1035 NULL, 1036 }; 1037 1038 static struct attribute *pstate_global_attributes[] = { 1039 &status.attr, 1040 NULL 1041 }; 1042 1043 static const struct attribute_group amd_pstate_global_attr_group = { 1044 .name = "amd_pstate", 1045 .attrs = pstate_global_attributes, 1046 }; 1047 1048 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1049 { 1050 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 1051 struct amd_cpudata *cpudata; 1052 struct device *dev; 1053 u64 value; 1054 1055 /* 1056 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1057 * which is ideal for initialization process. 1058 */ 1059 amd_perf_ctl_reset(policy->cpu); 1060 dev = get_cpu_device(policy->cpu); 1061 if (!dev) 1062 return -ENODEV; 1063 1064 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 1065 if (!cpudata) 1066 return -ENOMEM; 1067 1068 cpudata->cpu = policy->cpu; 1069 cpudata->epp_policy = 0; 1070 1071 ret = amd_pstate_init_perf(cpudata); 1072 if (ret) 1073 goto free_cpudata1; 1074 1075 min_freq = amd_get_min_freq(cpudata); 1076 max_freq = amd_get_max_freq(cpudata); 1077 nominal_freq = amd_get_nominal_freq(cpudata); 1078 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 1079 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 1080 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 1081 min_freq, max_freq); 1082 ret = -EINVAL; 1083 goto free_cpudata1; 1084 } 1085 1086 policy->cpuinfo.min_freq = min_freq; 1087 policy->cpuinfo.max_freq = max_freq; 1088 /* It will be updated by governor */ 1089 policy->cur = policy->cpuinfo.min_freq; 1090 1091 /* Initial processor data capability frequencies */ 1092 cpudata->max_freq = max_freq; 1093 cpudata->min_freq = min_freq; 1094 cpudata->nominal_freq = nominal_freq; 1095 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 1096 1097 policy->driver_data = cpudata; 1098 1099 cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); 1100 1101 policy->min = policy->cpuinfo.min_freq; 1102 policy->max = policy->cpuinfo.max_freq; 1103 1104 /* 1105 * Set the policy to powersave to provide a valid fallback value in case 1106 * the default cpufreq governor is neither powersave nor performance. 1107 */ 1108 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1109 1110 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1111 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 1112 if (ret) 1113 return ret; 1114 WRITE_ONCE(cpudata->cppc_req_cached, value); 1115 1116 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); 1117 if (ret) 1118 return ret; 1119 WRITE_ONCE(cpudata->cppc_cap1_cached, value); 1120 } 1121 amd_pstate_boost_init(cpudata); 1122 1123 return 0; 1124 1125 free_cpudata1: 1126 kfree(cpudata); 1127 return ret; 1128 } 1129 1130 static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1131 { 1132 pr_debug("CPU %d exiting\n", policy->cpu); 1133 return 0; 1134 } 1135 1136 static void amd_pstate_epp_init(unsigned int cpu) 1137 { 1138 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1139 struct amd_cpudata *cpudata = policy->driver_data; 1140 u32 max_perf, min_perf; 1141 u64 value; 1142 s16 epp; 1143 1144 max_perf = READ_ONCE(cpudata->highest_perf); 1145 min_perf = READ_ONCE(cpudata->lowest_perf); 1146 1147 value = READ_ONCE(cpudata->cppc_req_cached); 1148 1149 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1150 min_perf = max_perf; 1151 1152 /* Initial min/max values for CPPC Performance Controls Register */ 1153 value &= ~AMD_CPPC_MIN_PERF(~0L); 1154 value |= AMD_CPPC_MIN_PERF(min_perf); 1155 1156 value &= ~AMD_CPPC_MAX_PERF(~0L); 1157 value |= AMD_CPPC_MAX_PERF(max_perf); 1158 1159 /* CPPC EPP feature require to set zero to the desire perf bit */ 1160 value &= ~AMD_CPPC_DES_PERF(~0L); 1161 value |= AMD_CPPC_DES_PERF(0); 1162 1163 if (cpudata->epp_policy == cpudata->policy) 1164 goto skip_epp; 1165 1166 cpudata->epp_policy = cpudata->policy; 1167 1168 /* Get BIOS pre-defined epp value */ 1169 epp = amd_pstate_get_epp(cpudata, value); 1170 if (epp < 0) { 1171 /** 1172 * This return value can only be negative for shared_memory 1173 * systems where EPP register read/write not supported. 1174 */ 1175 goto skip_epp; 1176 } 1177 1178 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1179 epp = 0; 1180 1181 /* Set initial EPP value */ 1182 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1183 value &= ~GENMASK_ULL(31, 24); 1184 value |= (u64)epp << 24; 1185 } 1186 1187 WRITE_ONCE(cpudata->cppc_req_cached, value); 1188 amd_pstate_set_epp(cpudata, epp); 1189 skip_epp: 1190 cpufreq_cpu_put(policy); 1191 } 1192 1193 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 1194 { 1195 struct amd_cpudata *cpudata = policy->driver_data; 1196 1197 if (!policy->cpuinfo.max_freq) 1198 return -ENODEV; 1199 1200 pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", 1201 policy->cpuinfo.max_freq, policy->max); 1202 1203 cpudata->policy = policy->policy; 1204 1205 amd_pstate_epp_init(policy->cpu); 1206 1207 return 0; 1208 } 1209 1210 static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) 1211 { 1212 struct cppc_perf_ctrls perf_ctrls; 1213 u64 value, max_perf; 1214 int ret; 1215 1216 ret = amd_pstate_enable(true); 1217 if (ret) 1218 pr_err("failed to enable amd pstate during resume, return %d\n", ret); 1219 1220 value = READ_ONCE(cpudata->cppc_req_cached); 1221 max_perf = READ_ONCE(cpudata->highest_perf); 1222 1223 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1224 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1225 } else { 1226 perf_ctrls.max_perf = max_perf; 1227 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); 1228 cppc_set_perf(cpudata->cpu, &perf_ctrls); 1229 } 1230 } 1231 1232 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) 1233 { 1234 struct amd_cpudata *cpudata = policy->driver_data; 1235 1236 pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); 1237 1238 if (cppc_state == AMD_PSTATE_ACTIVE) { 1239 amd_pstate_epp_reenable(cpudata); 1240 cpudata->suspended = false; 1241 } 1242 1243 return 0; 1244 } 1245 1246 static void amd_pstate_epp_offline(struct cpufreq_policy *policy) 1247 { 1248 struct amd_cpudata *cpudata = policy->driver_data; 1249 struct cppc_perf_ctrls perf_ctrls; 1250 int min_perf; 1251 u64 value; 1252 1253 min_perf = READ_ONCE(cpudata->lowest_perf); 1254 value = READ_ONCE(cpudata->cppc_req_cached); 1255 1256 mutex_lock(&amd_pstate_limits_lock); 1257 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1258 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; 1259 1260 /* Set max perf same as min perf */ 1261 value &= ~AMD_CPPC_MAX_PERF(~0L); 1262 value |= AMD_CPPC_MAX_PERF(min_perf); 1263 value &= ~AMD_CPPC_MIN_PERF(~0L); 1264 value |= AMD_CPPC_MIN_PERF(min_perf); 1265 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1266 } else { 1267 perf_ctrls.desired_perf = 0; 1268 perf_ctrls.max_perf = min_perf; 1269 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); 1270 cppc_set_perf(cpudata->cpu, &perf_ctrls); 1271 } 1272 mutex_unlock(&amd_pstate_limits_lock); 1273 } 1274 1275 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) 1276 { 1277 struct amd_cpudata *cpudata = policy->driver_data; 1278 1279 pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); 1280 1281 if (cpudata->suspended) 1282 return 0; 1283 1284 if (cppc_state == AMD_PSTATE_ACTIVE) 1285 amd_pstate_epp_offline(policy); 1286 1287 return 0; 1288 } 1289 1290 static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) 1291 { 1292 cpufreq_verify_within_cpu_limits(policy); 1293 pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); 1294 return 0; 1295 } 1296 1297 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) 1298 { 1299 struct amd_cpudata *cpudata = policy->driver_data; 1300 int ret; 1301 1302 /* avoid suspending when EPP is not enabled */ 1303 if (cppc_state != AMD_PSTATE_ACTIVE) 1304 return 0; 1305 1306 /* set this flag to avoid setting core offline*/ 1307 cpudata->suspended = true; 1308 1309 /* disable CPPC in lowlevel firmware */ 1310 ret = amd_pstate_enable(false); 1311 if (ret) 1312 pr_err("failed to suspend, return %d\n", ret); 1313 1314 return 0; 1315 } 1316 1317 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 1318 { 1319 struct amd_cpudata *cpudata = policy->driver_data; 1320 1321 if (cpudata->suspended) { 1322 mutex_lock(&amd_pstate_limits_lock); 1323 1324 /* enable amd pstate from suspend state*/ 1325 amd_pstate_epp_reenable(cpudata); 1326 1327 mutex_unlock(&amd_pstate_limits_lock); 1328 1329 cpudata->suspended = false; 1330 } 1331 1332 return 0; 1333 } 1334 1335 static struct cpufreq_driver amd_pstate_driver = { 1336 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 1337 .verify = amd_pstate_verify, 1338 .target = amd_pstate_target, 1339 .fast_switch = amd_pstate_fast_switch, 1340 .init = amd_pstate_cpu_init, 1341 .exit = amd_pstate_cpu_exit, 1342 .suspend = amd_pstate_cpu_suspend, 1343 .resume = amd_pstate_cpu_resume, 1344 .set_boost = amd_pstate_set_boost, 1345 .name = "amd-pstate", 1346 .attr = amd_pstate_attr, 1347 }; 1348 1349 static struct cpufreq_driver amd_pstate_epp_driver = { 1350 .flags = CPUFREQ_CONST_LOOPS, 1351 .verify = amd_pstate_epp_verify_policy, 1352 .setpolicy = amd_pstate_epp_set_policy, 1353 .init = amd_pstate_epp_cpu_init, 1354 .exit = amd_pstate_epp_cpu_exit, 1355 .offline = amd_pstate_epp_cpu_offline, 1356 .online = amd_pstate_epp_cpu_online, 1357 .suspend = amd_pstate_epp_suspend, 1358 .resume = amd_pstate_epp_resume, 1359 .name = "amd_pstate_epp", 1360 .attr = amd_pstate_epp_attr, 1361 }; 1362 1363 static int __init amd_pstate_init(void) 1364 { 1365 struct device *dev_root; 1366 int ret; 1367 1368 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 1369 return -ENODEV; 1370 /* 1371 * by default the pstate driver is disabled to load 1372 * enable the amd_pstate passive mode driver explicitly 1373 * with amd_pstate=passive or other modes in kernel command line 1374 */ 1375 if (cppc_state == AMD_PSTATE_DISABLE) { 1376 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 1377 return -ENODEV; 1378 } 1379 1380 if (!acpi_cpc_valid()) { 1381 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 1382 return -ENODEV; 1383 } 1384 1385 /* don't keep reloading if cpufreq_driver exists */ 1386 if (cpufreq_get_current_driver()) 1387 return -EEXIST; 1388 1389 /* capability check */ 1390 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1391 pr_debug("AMD CPPC MSR based functionality is supported\n"); 1392 if (cppc_state != AMD_PSTATE_ACTIVE) 1393 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1394 } else { 1395 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 1396 static_call_update(amd_pstate_enable, cppc_enable); 1397 static_call_update(amd_pstate_init_perf, cppc_init_perf); 1398 static_call_update(amd_pstate_update_perf, cppc_update_perf); 1399 } 1400 1401 /* enable amd pstate feature */ 1402 ret = amd_pstate_enable(true); 1403 if (ret) { 1404 pr_err("failed to enable with return %d\n", ret); 1405 return ret; 1406 } 1407 1408 ret = cpufreq_register_driver(current_pstate_driver); 1409 if (ret) 1410 pr_err("failed to register with return %d\n", ret); 1411 1412 dev_root = bus_get_dev_root(&cpu_subsys); 1413 if (dev_root) { 1414 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); 1415 put_device(dev_root); 1416 if (ret) { 1417 pr_err("sysfs attribute export failed with error %d.\n", ret); 1418 goto global_attr_free; 1419 } 1420 } 1421 1422 return ret; 1423 1424 global_attr_free: 1425 cpufreq_unregister_driver(current_pstate_driver); 1426 return ret; 1427 } 1428 device_initcall(amd_pstate_init); 1429 1430 static int __init amd_pstate_param(char *str) 1431 { 1432 size_t size; 1433 int mode_idx; 1434 1435 if (!str) 1436 return -EINVAL; 1437 1438 size = strlen(str); 1439 mode_idx = get_mode_idx_from_str(str, size); 1440 1441 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1442 cppc_state = mode_idx; 1443 if (cppc_state == AMD_PSTATE_DISABLE) 1444 pr_info("driver is explicitly disabled\n"); 1445 1446 if (cppc_state == AMD_PSTATE_ACTIVE) 1447 current_pstate_driver = &amd_pstate_epp_driver; 1448 1449 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1450 current_pstate_driver = &amd_pstate_driver; 1451 1452 return 0; 1453 } 1454 1455 return -EINVAL; 1456 } 1457 early_param("amd_pstate", amd_pstate_param); 1458 1459 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 1460 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 1461