1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/kernel.h> 26 #include <linux/module.h> 27 #include <linux/init.h> 28 #include <linux/smp.h> 29 #include <linux/sched.h> 30 #include <linux/cpufreq.h> 31 #include <linux/compiler.h> 32 #include <linux/dmi.h> 33 #include <linux/slab.h> 34 #include <linux/acpi.h> 35 #include <linux/io.h> 36 #include <linux/delay.h> 37 #include <linux/uaccess.h> 38 #include <linux/static_call.h> 39 #include <linux/amd-pstate.h> 40 41 #include <acpi/processor.h> 42 #include <acpi/cppc_acpi.h> 43 44 #include <asm/msr.h> 45 #include <asm/processor.h> 46 #include <asm/cpufeature.h> 47 #include <asm/cpu_device_id.h> 48 #include "amd-pstate-trace.h" 49 50 #define AMD_PSTATE_TRANSITION_LATENCY 20000 51 #define AMD_PSTATE_TRANSITION_DELAY 1000 52 53 /* 54 * TODO: We need more time to fine tune processors with shared memory solution 55 * with community together. 56 * 57 * There are some performance drops on the CPU benchmarks which reports from 58 * Suse. We are co-working with them to fine tune the shared memory solution. So 59 * we disable it by default to go acpi-cpufreq on these processors and add a 60 * module parameter to be able to enable it manually for debugging. 61 */ 62 static struct cpufreq_driver *current_pstate_driver; 63 static struct cpufreq_driver amd_pstate_driver; 64 static struct cpufreq_driver amd_pstate_epp_driver; 65 static int cppc_state = AMD_PSTATE_DISABLE; 66 67 /* 68 * AMD Energy Preference Performance (EPP) 69 * The EPP is used in the CCLK DPM controller to drive 70 * the frequency that a core is going to operate during 71 * short periods of activity. EPP values will be utilized for 72 * different OS profiles (balanced, performance, power savings) 73 * display strings corresponding to EPP index in the 74 * energy_perf_strings[] 75 * index String 76 *------------------------------------- 77 * 0 default 78 * 1 performance 79 * 2 balance_performance 80 * 3 balance_power 81 * 4 power 82 */ 83 enum energy_perf_value_index { 84 EPP_INDEX_DEFAULT = 0, 85 EPP_INDEX_PERFORMANCE, 86 EPP_INDEX_BALANCE_PERFORMANCE, 87 EPP_INDEX_BALANCE_POWERSAVE, 88 EPP_INDEX_POWERSAVE, 89 }; 90 91 static const char * const energy_perf_strings[] = { 92 [EPP_INDEX_DEFAULT] = "default", 93 [EPP_INDEX_PERFORMANCE] = "performance", 94 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 95 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 96 [EPP_INDEX_POWERSAVE] = "power", 97 NULL 98 }; 99 100 static unsigned int epp_values[] = { 101 [EPP_INDEX_DEFAULT] = 0, 102 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 103 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 104 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 105 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 106 }; 107 108 typedef int (*cppc_mode_transition_fn)(int); 109 110 static inline int get_mode_idx_from_str(const char *str, size_t size) 111 { 112 int i; 113 114 for (i=0; i < AMD_PSTATE_MAX; i++) { 115 if (!strncmp(str, amd_pstate_mode_string[i], size)) 116 return i; 117 } 118 return -EINVAL; 119 } 120 121 static DEFINE_MUTEX(amd_pstate_limits_lock); 122 static DEFINE_MUTEX(amd_pstate_driver_lock); 123 124 static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) 125 { 126 u64 epp; 127 int ret; 128 129 if (boot_cpu_has(X86_FEATURE_CPPC)) { 130 if (!cppc_req_cached) { 131 epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 132 &cppc_req_cached); 133 if (epp) 134 return epp; 135 } 136 epp = (cppc_req_cached >> 24) & 0xFF; 137 } else { 138 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 139 if (ret < 0) { 140 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 141 return -EIO; 142 } 143 } 144 145 return (s16)(epp & 0xff); 146 } 147 148 static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) 149 { 150 s16 epp; 151 int index = -EINVAL; 152 153 epp = amd_pstate_get_epp(cpudata, 0); 154 if (epp < 0) 155 return epp; 156 157 switch (epp) { 158 case AMD_CPPC_EPP_PERFORMANCE: 159 index = EPP_INDEX_PERFORMANCE; 160 break; 161 case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 162 index = EPP_INDEX_BALANCE_PERFORMANCE; 163 break; 164 case AMD_CPPC_EPP_BALANCE_POWERSAVE: 165 index = EPP_INDEX_BALANCE_POWERSAVE; 166 break; 167 case AMD_CPPC_EPP_POWERSAVE: 168 index = EPP_INDEX_POWERSAVE; 169 break; 170 default: 171 break; 172 } 173 174 return index; 175 } 176 177 static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) 178 { 179 int ret; 180 struct cppc_perf_ctrls perf_ctrls; 181 182 if (boot_cpu_has(X86_FEATURE_CPPC)) { 183 u64 value = READ_ONCE(cpudata->cppc_req_cached); 184 185 value &= ~GENMASK_ULL(31, 24); 186 value |= (u64)epp << 24; 187 WRITE_ONCE(cpudata->cppc_req_cached, value); 188 189 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 190 if (!ret) 191 cpudata->epp_cached = epp; 192 } else { 193 perf_ctrls.energy_perf = epp; 194 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 195 if (ret) { 196 pr_debug("failed to set energy perf value (%d)\n", ret); 197 return ret; 198 } 199 cpudata->epp_cached = epp; 200 } 201 202 return ret; 203 } 204 205 static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, 206 int pref_index) 207 { 208 int epp = -EINVAL; 209 int ret; 210 211 if (!pref_index) { 212 pr_debug("EPP pref_index is invalid\n"); 213 return -EINVAL; 214 } 215 216 if (epp == -EINVAL) 217 epp = epp_values[pref_index]; 218 219 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 220 pr_debug("EPP cannot be set under performance policy\n"); 221 return -EBUSY; 222 } 223 224 ret = amd_pstate_set_epp(cpudata, epp); 225 226 return ret; 227 } 228 229 static inline int pstate_enable(bool enable) 230 { 231 return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); 232 } 233 234 static int cppc_enable(bool enable) 235 { 236 int cpu, ret = 0; 237 struct cppc_perf_ctrls perf_ctrls; 238 239 for_each_present_cpu(cpu) { 240 ret = cppc_set_enable(cpu, enable); 241 if (ret) 242 return ret; 243 244 /* Enable autonomous mode for EPP */ 245 if (cppc_state == AMD_PSTATE_ACTIVE) { 246 /* Set desired perf as zero to allow EPP firmware control */ 247 perf_ctrls.desired_perf = 0; 248 ret = cppc_set_perf(cpu, &perf_ctrls); 249 if (ret) 250 return ret; 251 } 252 } 253 254 return ret; 255 } 256 257 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); 258 259 static inline int amd_pstate_enable(bool enable) 260 { 261 return static_call(amd_pstate_enable)(enable); 262 } 263 264 static int pstate_init_perf(struct amd_cpudata *cpudata) 265 { 266 u64 cap1; 267 u32 highest_perf; 268 269 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 270 &cap1); 271 if (ret) 272 return ret; 273 274 /* 275 * TODO: Introduce AMD specific power feature. 276 * 277 * CPPC entry doesn't indicate the highest performance in some ASICs. 278 */ 279 highest_perf = amd_get_highest_perf(); 280 if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) 281 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); 282 283 WRITE_ONCE(cpudata->highest_perf, highest_perf); 284 285 WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); 286 WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); 287 WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); 288 289 return 0; 290 } 291 292 static int cppc_init_perf(struct amd_cpudata *cpudata) 293 { 294 struct cppc_perf_caps cppc_perf; 295 u32 highest_perf; 296 297 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 298 if (ret) 299 return ret; 300 301 highest_perf = amd_get_highest_perf(); 302 if (highest_perf > cppc_perf.highest_perf) 303 highest_perf = cppc_perf.highest_perf; 304 305 WRITE_ONCE(cpudata->highest_perf, highest_perf); 306 307 WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); 308 WRITE_ONCE(cpudata->lowest_nonlinear_perf, 309 cppc_perf.lowest_nonlinear_perf); 310 WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); 311 312 if (cppc_state == AMD_PSTATE_ACTIVE) 313 return 0; 314 315 ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf); 316 if (ret) { 317 pr_warn("failed to get auto_sel, ret: %d\n", ret); 318 return 0; 319 } 320 321 ret = cppc_set_auto_sel(cpudata->cpu, 322 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 323 324 if (ret) 325 pr_warn("failed to set auto_sel, ret: %d\n", ret); 326 327 return ret; 328 } 329 330 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); 331 332 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 333 { 334 return static_call(amd_pstate_init_perf)(cpudata); 335 } 336 337 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, 338 u32 des_perf, u32 max_perf, bool fast_switch) 339 { 340 if (fast_switch) 341 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); 342 else 343 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 344 READ_ONCE(cpudata->cppc_req_cached)); 345 } 346 347 static void cppc_update_perf(struct amd_cpudata *cpudata, 348 u32 min_perf, u32 des_perf, 349 u32 max_perf, bool fast_switch) 350 { 351 struct cppc_perf_ctrls perf_ctrls; 352 353 perf_ctrls.max_perf = max_perf; 354 perf_ctrls.min_perf = min_perf; 355 perf_ctrls.desired_perf = des_perf; 356 357 cppc_set_perf(cpudata->cpu, &perf_ctrls); 358 } 359 360 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); 361 362 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, 363 u32 min_perf, u32 des_perf, 364 u32 max_perf, bool fast_switch) 365 { 366 static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, 367 max_perf, fast_switch); 368 } 369 370 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 371 { 372 u64 aperf, mperf, tsc; 373 unsigned long flags; 374 375 local_irq_save(flags); 376 rdmsrl(MSR_IA32_APERF, aperf); 377 rdmsrl(MSR_IA32_MPERF, mperf); 378 tsc = rdtsc(); 379 380 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 381 local_irq_restore(flags); 382 return false; 383 } 384 385 local_irq_restore(flags); 386 387 cpudata->cur.aperf = aperf; 388 cpudata->cur.mperf = mperf; 389 cpudata->cur.tsc = tsc; 390 cpudata->cur.aperf -= cpudata->prev.aperf; 391 cpudata->cur.mperf -= cpudata->prev.mperf; 392 cpudata->cur.tsc -= cpudata->prev.tsc; 393 394 cpudata->prev.aperf = aperf; 395 cpudata->prev.mperf = mperf; 396 cpudata->prev.tsc = tsc; 397 398 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 399 400 return true; 401 } 402 403 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, 404 u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags) 405 { 406 u64 prev = READ_ONCE(cpudata->cppc_req_cached); 407 u64 value = prev; 408 409 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 410 411 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 412 min_perf = des_perf; 413 des_perf = 0; 414 } 415 416 value &= ~AMD_CPPC_MIN_PERF(~0L); 417 value |= AMD_CPPC_MIN_PERF(min_perf); 418 419 value &= ~AMD_CPPC_DES_PERF(~0L); 420 value |= AMD_CPPC_DES_PERF(des_perf); 421 422 value &= ~AMD_CPPC_MAX_PERF(~0L); 423 value |= AMD_CPPC_MAX_PERF(max_perf); 424 425 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 426 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 427 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 428 cpudata->cpu, (value != prev), fast_switch); 429 } 430 431 if (value == prev) 432 return; 433 434 WRITE_ONCE(cpudata->cppc_req_cached, value); 435 436 amd_pstate_update_perf(cpudata, min_perf, des_perf, 437 max_perf, fast_switch); 438 } 439 440 static int amd_pstate_verify(struct cpufreq_policy_data *policy) 441 { 442 cpufreq_verify_within_cpu_limits(policy); 443 444 return 0; 445 } 446 447 static int amd_pstate_update_freq(struct cpufreq_policy *policy, 448 unsigned int target_freq, bool fast_switch) 449 { 450 struct cpufreq_freqs freqs; 451 struct amd_cpudata *cpudata = policy->driver_data; 452 unsigned long max_perf, min_perf, des_perf, cap_perf; 453 454 if (!cpudata->max_freq) 455 return -ENODEV; 456 457 cap_perf = READ_ONCE(cpudata->highest_perf); 458 min_perf = READ_ONCE(cpudata->lowest_perf); 459 max_perf = cap_perf; 460 461 freqs.old = policy->cur; 462 freqs.new = target_freq; 463 464 des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, 465 cpudata->max_freq); 466 467 WARN_ON(fast_switch && !policy->fast_switch_enabled); 468 /* 469 * If fast_switch is desired, then there aren't any registered 470 * transition notifiers. See comment for 471 * cpufreq_enable_fast_switch(). 472 */ 473 if (!fast_switch) 474 cpufreq_freq_transition_begin(policy, &freqs); 475 476 amd_pstate_update(cpudata, min_perf, des_perf, 477 max_perf, fast_switch, policy->governor->flags); 478 479 if (!fast_switch) 480 cpufreq_freq_transition_end(policy, &freqs, false); 481 482 return 0; 483 } 484 485 static int amd_pstate_target(struct cpufreq_policy *policy, 486 unsigned int target_freq, 487 unsigned int relation) 488 { 489 return amd_pstate_update_freq(policy, target_freq, false); 490 } 491 492 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, 493 unsigned int target_freq) 494 { 495 return amd_pstate_update_freq(policy, target_freq, true); 496 } 497 498 static void amd_pstate_adjust_perf(unsigned int cpu, 499 unsigned long _min_perf, 500 unsigned long target_perf, 501 unsigned long capacity) 502 { 503 unsigned long max_perf, min_perf, des_perf, 504 cap_perf, lowest_nonlinear_perf; 505 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 506 struct amd_cpudata *cpudata = policy->driver_data; 507 508 cap_perf = READ_ONCE(cpudata->highest_perf); 509 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); 510 511 des_perf = cap_perf; 512 if (target_perf < capacity) 513 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 514 515 min_perf = READ_ONCE(cpudata->highest_perf); 516 if (_min_perf < capacity) 517 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 518 519 if (min_perf < lowest_nonlinear_perf) 520 min_perf = lowest_nonlinear_perf; 521 522 max_perf = cap_perf; 523 if (max_perf < min_perf) 524 max_perf = min_perf; 525 526 amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, 527 policy->governor->flags); 528 cpufreq_cpu_put(policy); 529 } 530 531 static int amd_get_min_freq(struct amd_cpudata *cpudata) 532 { 533 struct cppc_perf_caps cppc_perf; 534 535 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 536 if (ret) 537 return ret; 538 539 /* Switch to khz */ 540 return cppc_perf.lowest_freq * 1000; 541 } 542 543 static int amd_get_max_freq(struct amd_cpudata *cpudata) 544 { 545 struct cppc_perf_caps cppc_perf; 546 u32 max_perf, max_freq, nominal_freq, nominal_perf; 547 u64 boost_ratio; 548 549 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 550 if (ret) 551 return ret; 552 553 nominal_freq = cppc_perf.nominal_freq; 554 nominal_perf = READ_ONCE(cpudata->nominal_perf); 555 max_perf = READ_ONCE(cpudata->highest_perf); 556 557 boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT, 558 nominal_perf); 559 560 max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT; 561 562 /* Switch to khz */ 563 return max_freq * 1000; 564 } 565 566 static int amd_get_nominal_freq(struct amd_cpudata *cpudata) 567 { 568 struct cppc_perf_caps cppc_perf; 569 570 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 571 if (ret) 572 return ret; 573 574 /* Switch to khz */ 575 return cppc_perf.nominal_freq * 1000; 576 } 577 578 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) 579 { 580 struct cppc_perf_caps cppc_perf; 581 u32 lowest_nonlinear_freq, lowest_nonlinear_perf, 582 nominal_freq, nominal_perf; 583 u64 lowest_nonlinear_ratio; 584 585 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 586 if (ret) 587 return ret; 588 589 nominal_freq = cppc_perf.nominal_freq; 590 nominal_perf = READ_ONCE(cpudata->nominal_perf); 591 592 lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 593 594 lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, 595 nominal_perf); 596 597 lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT; 598 599 /* Switch to khz */ 600 return lowest_nonlinear_freq * 1000; 601 } 602 603 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 604 { 605 struct amd_cpudata *cpudata = policy->driver_data; 606 int ret; 607 608 if (!cpudata->boost_supported) { 609 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 610 return -EINVAL; 611 } 612 613 if (state) 614 policy->cpuinfo.max_freq = cpudata->max_freq; 615 else 616 policy->cpuinfo.max_freq = cpudata->nominal_freq; 617 618 policy->max = policy->cpuinfo.max_freq; 619 620 ret = freq_qos_update_request(&cpudata->req[1], 621 policy->cpuinfo.max_freq); 622 if (ret < 0) 623 return ret; 624 625 return 0; 626 } 627 628 static void amd_pstate_boost_init(struct amd_cpudata *cpudata) 629 { 630 u32 highest_perf, nominal_perf; 631 632 highest_perf = READ_ONCE(cpudata->highest_perf); 633 nominal_perf = READ_ONCE(cpudata->nominal_perf); 634 635 if (highest_perf <= nominal_perf) 636 return; 637 638 cpudata->boost_supported = true; 639 current_pstate_driver->boost_enabled = true; 640 } 641 642 static void amd_perf_ctl_reset(unsigned int cpu) 643 { 644 wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 645 } 646 647 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 648 { 649 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 650 struct device *dev; 651 struct amd_cpudata *cpudata; 652 653 /* 654 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 655 * which is ideal for initialization process. 656 */ 657 amd_perf_ctl_reset(policy->cpu); 658 dev = get_cpu_device(policy->cpu); 659 if (!dev) 660 return -ENODEV; 661 662 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 663 if (!cpudata) 664 return -ENOMEM; 665 666 cpudata->cpu = policy->cpu; 667 668 ret = amd_pstate_init_perf(cpudata); 669 if (ret) 670 goto free_cpudata1; 671 672 min_freq = amd_get_min_freq(cpudata); 673 max_freq = amd_get_max_freq(cpudata); 674 nominal_freq = amd_get_nominal_freq(cpudata); 675 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 676 677 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 678 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 679 min_freq, max_freq); 680 ret = -EINVAL; 681 goto free_cpudata1; 682 } 683 684 policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; 685 policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; 686 687 policy->min = min_freq; 688 policy->max = max_freq; 689 690 policy->cpuinfo.min_freq = min_freq; 691 policy->cpuinfo.max_freq = max_freq; 692 693 /* It will be updated by governor */ 694 policy->cur = policy->cpuinfo.min_freq; 695 696 if (boot_cpu_has(X86_FEATURE_CPPC)) 697 policy->fast_switch_possible = true; 698 699 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 700 FREQ_QOS_MIN, policy->cpuinfo.min_freq); 701 if (ret < 0) { 702 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 703 goto free_cpudata1; 704 } 705 706 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 707 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 708 if (ret < 0) { 709 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 710 goto free_cpudata2; 711 } 712 713 /* Initial processor data capability frequencies */ 714 cpudata->max_freq = max_freq; 715 cpudata->min_freq = min_freq; 716 cpudata->nominal_freq = nominal_freq; 717 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 718 719 policy->driver_data = cpudata; 720 721 amd_pstate_boost_init(cpudata); 722 if (!current_pstate_driver->adjust_perf) 723 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 724 725 return 0; 726 727 free_cpudata2: 728 freq_qos_remove_request(&cpudata->req[0]); 729 free_cpudata1: 730 kfree(cpudata); 731 return ret; 732 } 733 734 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) 735 { 736 struct amd_cpudata *cpudata = policy->driver_data; 737 738 freq_qos_remove_request(&cpudata->req[1]); 739 freq_qos_remove_request(&cpudata->req[0]); 740 policy->fast_switch_possible = false; 741 kfree(cpudata); 742 743 return 0; 744 } 745 746 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) 747 { 748 int ret; 749 750 ret = amd_pstate_enable(true); 751 if (ret) 752 pr_err("failed to enable amd-pstate during resume, return %d\n", ret); 753 754 return ret; 755 } 756 757 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) 758 { 759 int ret; 760 761 ret = amd_pstate_enable(false); 762 if (ret) 763 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); 764 765 return ret; 766 } 767 768 /* Sysfs attributes */ 769 770 /* 771 * This frequency is to indicate the maximum hardware frequency. 772 * If boost is not active but supported, the frequency will be larger than the 773 * one in cpuinfo. 774 */ 775 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 776 char *buf) 777 { 778 int max_freq; 779 struct amd_cpudata *cpudata = policy->driver_data; 780 781 max_freq = amd_get_max_freq(cpudata); 782 if (max_freq < 0) 783 return max_freq; 784 785 return sysfs_emit(buf, "%u\n", max_freq); 786 } 787 788 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 789 char *buf) 790 { 791 int freq; 792 struct amd_cpudata *cpudata = policy->driver_data; 793 794 freq = amd_get_lowest_nonlinear_freq(cpudata); 795 if (freq < 0) 796 return freq; 797 798 return sysfs_emit(buf, "%u\n", freq); 799 } 800 801 /* 802 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 803 * need to expose it to sysfs. 804 */ 805 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 806 char *buf) 807 { 808 u32 perf; 809 struct amd_cpudata *cpudata = policy->driver_data; 810 811 perf = READ_ONCE(cpudata->highest_perf); 812 813 return sysfs_emit(buf, "%u\n", perf); 814 } 815 816 static ssize_t show_energy_performance_available_preferences( 817 struct cpufreq_policy *policy, char *buf) 818 { 819 int i = 0; 820 int offset = 0; 821 822 while (energy_perf_strings[i] != NULL) 823 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); 824 825 sysfs_emit_at(buf, offset, "\n"); 826 827 return offset; 828 } 829 830 static ssize_t store_energy_performance_preference( 831 struct cpufreq_policy *policy, const char *buf, size_t count) 832 { 833 struct amd_cpudata *cpudata = policy->driver_data; 834 char str_preference[21]; 835 ssize_t ret; 836 837 ret = sscanf(buf, "%20s", str_preference); 838 if (ret != 1) 839 return -EINVAL; 840 841 ret = match_string(energy_perf_strings, -1, str_preference); 842 if (ret < 0) 843 return -EINVAL; 844 845 mutex_lock(&amd_pstate_limits_lock); 846 ret = amd_pstate_set_energy_pref_index(cpudata, ret); 847 mutex_unlock(&amd_pstate_limits_lock); 848 849 return ret ?: count; 850 } 851 852 static ssize_t show_energy_performance_preference( 853 struct cpufreq_policy *policy, char *buf) 854 { 855 struct amd_cpudata *cpudata = policy->driver_data; 856 int preference; 857 858 preference = amd_pstate_get_energy_pref_index(cpudata); 859 if (preference < 0) 860 return preference; 861 862 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 863 } 864 865 static void amd_pstate_driver_cleanup(void) 866 { 867 amd_pstate_enable(false); 868 cppc_state = AMD_PSTATE_DISABLE; 869 current_pstate_driver = NULL; 870 } 871 872 static int amd_pstate_register_driver(int mode) 873 { 874 int ret; 875 876 if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED) 877 current_pstate_driver = &amd_pstate_driver; 878 else if (mode == AMD_PSTATE_ACTIVE) 879 current_pstate_driver = &amd_pstate_epp_driver; 880 else 881 return -EINVAL; 882 883 cppc_state = mode; 884 ret = cpufreq_register_driver(current_pstate_driver); 885 if (ret) { 886 amd_pstate_driver_cleanup(); 887 return ret; 888 } 889 return 0; 890 } 891 892 static int amd_pstate_unregister_driver(int dummy) 893 { 894 cpufreq_unregister_driver(current_pstate_driver); 895 amd_pstate_driver_cleanup(); 896 return 0; 897 } 898 899 static int amd_pstate_change_mode_without_dvr_change(int mode) 900 { 901 int cpu = 0; 902 903 cppc_state = mode; 904 905 if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 906 return 0; 907 908 for_each_present_cpu(cpu) { 909 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 910 } 911 912 return 0; 913 } 914 915 static int amd_pstate_change_driver_mode(int mode) 916 { 917 int ret; 918 919 ret = amd_pstate_unregister_driver(0); 920 if (ret) 921 return ret; 922 923 ret = amd_pstate_register_driver(mode); 924 if (ret) 925 return ret; 926 927 return 0; 928 } 929 930 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 931 [AMD_PSTATE_DISABLE] = { 932 [AMD_PSTATE_DISABLE] = NULL, 933 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 934 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 935 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 936 }, 937 [AMD_PSTATE_PASSIVE] = { 938 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 939 [AMD_PSTATE_PASSIVE] = NULL, 940 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 941 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 942 }, 943 [AMD_PSTATE_ACTIVE] = { 944 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 945 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 946 [AMD_PSTATE_ACTIVE] = NULL, 947 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 948 }, 949 [AMD_PSTATE_GUIDED] = { 950 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 951 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 952 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 953 [AMD_PSTATE_GUIDED] = NULL, 954 }, 955 }; 956 957 static ssize_t amd_pstate_show_status(char *buf) 958 { 959 if (!current_pstate_driver) 960 return sysfs_emit(buf, "disable\n"); 961 962 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 963 } 964 965 static int amd_pstate_update_status(const char *buf, size_t size) 966 { 967 int mode_idx; 968 969 if (size > strlen("passive") || size < strlen("active")) 970 return -EINVAL; 971 972 mode_idx = get_mode_idx_from_str(buf, size); 973 974 if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX) 975 return -EINVAL; 976 977 if (mode_state_machine[cppc_state][mode_idx]) 978 return mode_state_machine[cppc_state][mode_idx](mode_idx); 979 980 return 0; 981 } 982 983 static ssize_t show_status(struct kobject *kobj, 984 struct kobj_attribute *attr, char *buf) 985 { 986 ssize_t ret; 987 988 mutex_lock(&amd_pstate_driver_lock); 989 ret = amd_pstate_show_status(buf); 990 mutex_unlock(&amd_pstate_driver_lock); 991 992 return ret; 993 } 994 995 static ssize_t store_status(struct kobject *a, struct kobj_attribute *b, 996 const char *buf, size_t count) 997 { 998 char *p = memchr(buf, '\n', count); 999 int ret; 1000 1001 mutex_lock(&amd_pstate_driver_lock); 1002 ret = amd_pstate_update_status(buf, p ? p - buf : count); 1003 mutex_unlock(&amd_pstate_driver_lock); 1004 1005 return ret < 0 ? ret : count; 1006 } 1007 1008 cpufreq_freq_attr_ro(amd_pstate_max_freq); 1009 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 1010 1011 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 1012 cpufreq_freq_attr_rw(energy_performance_preference); 1013 cpufreq_freq_attr_ro(energy_performance_available_preferences); 1014 define_one_global_rw(status); 1015 1016 static struct freq_attr *amd_pstate_attr[] = { 1017 &amd_pstate_max_freq, 1018 &amd_pstate_lowest_nonlinear_freq, 1019 &amd_pstate_highest_perf, 1020 NULL, 1021 }; 1022 1023 static struct freq_attr *amd_pstate_epp_attr[] = { 1024 &amd_pstate_max_freq, 1025 &amd_pstate_lowest_nonlinear_freq, 1026 &amd_pstate_highest_perf, 1027 &energy_performance_preference, 1028 &energy_performance_available_preferences, 1029 NULL, 1030 }; 1031 1032 static struct attribute *pstate_global_attributes[] = { 1033 &status.attr, 1034 NULL 1035 }; 1036 1037 static const struct attribute_group amd_pstate_global_attr_group = { 1038 .name = "amd_pstate", 1039 .attrs = pstate_global_attributes, 1040 }; 1041 1042 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1043 { 1044 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 1045 struct amd_cpudata *cpudata; 1046 struct device *dev; 1047 u64 value; 1048 1049 /* 1050 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1051 * which is ideal for initialization process. 1052 */ 1053 amd_perf_ctl_reset(policy->cpu); 1054 dev = get_cpu_device(policy->cpu); 1055 if (!dev) 1056 return -ENODEV; 1057 1058 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 1059 if (!cpudata) 1060 return -ENOMEM; 1061 1062 cpudata->cpu = policy->cpu; 1063 cpudata->epp_policy = 0; 1064 1065 ret = amd_pstate_init_perf(cpudata); 1066 if (ret) 1067 goto free_cpudata1; 1068 1069 min_freq = amd_get_min_freq(cpudata); 1070 max_freq = amd_get_max_freq(cpudata); 1071 nominal_freq = amd_get_nominal_freq(cpudata); 1072 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 1073 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 1074 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 1075 min_freq, max_freq); 1076 ret = -EINVAL; 1077 goto free_cpudata1; 1078 } 1079 1080 policy->cpuinfo.min_freq = min_freq; 1081 policy->cpuinfo.max_freq = max_freq; 1082 /* It will be updated by governor */ 1083 policy->cur = policy->cpuinfo.min_freq; 1084 1085 /* Initial processor data capability frequencies */ 1086 cpudata->max_freq = max_freq; 1087 cpudata->min_freq = min_freq; 1088 cpudata->nominal_freq = nominal_freq; 1089 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 1090 1091 policy->driver_data = cpudata; 1092 1093 cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); 1094 1095 policy->min = policy->cpuinfo.min_freq; 1096 policy->max = policy->cpuinfo.max_freq; 1097 1098 /* 1099 * Set the policy to powersave to provide a valid fallback value in case 1100 * the default cpufreq governor is neither powersave nor performance. 1101 */ 1102 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1103 1104 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1105 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 1106 if (ret) 1107 return ret; 1108 WRITE_ONCE(cpudata->cppc_req_cached, value); 1109 1110 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); 1111 if (ret) 1112 return ret; 1113 WRITE_ONCE(cpudata->cppc_cap1_cached, value); 1114 } 1115 amd_pstate_boost_init(cpudata); 1116 1117 return 0; 1118 1119 free_cpudata1: 1120 kfree(cpudata); 1121 return ret; 1122 } 1123 1124 static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1125 { 1126 pr_debug("CPU %d exiting\n", policy->cpu); 1127 return 0; 1128 } 1129 1130 static void amd_pstate_epp_init(unsigned int cpu) 1131 { 1132 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1133 struct amd_cpudata *cpudata = policy->driver_data; 1134 u32 max_perf, min_perf; 1135 u64 value; 1136 s16 epp; 1137 1138 max_perf = READ_ONCE(cpudata->highest_perf); 1139 min_perf = READ_ONCE(cpudata->lowest_perf); 1140 1141 value = READ_ONCE(cpudata->cppc_req_cached); 1142 1143 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1144 min_perf = max_perf; 1145 1146 /* Initial min/max values for CPPC Performance Controls Register */ 1147 value &= ~AMD_CPPC_MIN_PERF(~0L); 1148 value |= AMD_CPPC_MIN_PERF(min_perf); 1149 1150 value &= ~AMD_CPPC_MAX_PERF(~0L); 1151 value |= AMD_CPPC_MAX_PERF(max_perf); 1152 1153 /* CPPC EPP feature require to set zero to the desire perf bit */ 1154 value &= ~AMD_CPPC_DES_PERF(~0L); 1155 value |= AMD_CPPC_DES_PERF(0); 1156 1157 if (cpudata->epp_policy == cpudata->policy) 1158 goto skip_epp; 1159 1160 cpudata->epp_policy = cpudata->policy; 1161 1162 /* Get BIOS pre-defined epp value */ 1163 epp = amd_pstate_get_epp(cpudata, value); 1164 if (epp < 0) { 1165 /** 1166 * This return value can only be negative for shared_memory 1167 * systems where EPP register read/write not supported. 1168 */ 1169 goto skip_epp; 1170 } 1171 1172 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1173 epp = 0; 1174 1175 /* Set initial EPP value */ 1176 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1177 value &= ~GENMASK_ULL(31, 24); 1178 value |= (u64)epp << 24; 1179 } 1180 1181 WRITE_ONCE(cpudata->cppc_req_cached, value); 1182 amd_pstate_set_epp(cpudata, epp); 1183 skip_epp: 1184 cpufreq_cpu_put(policy); 1185 } 1186 1187 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 1188 { 1189 struct amd_cpudata *cpudata = policy->driver_data; 1190 1191 if (!policy->cpuinfo.max_freq) 1192 return -ENODEV; 1193 1194 pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", 1195 policy->cpuinfo.max_freq, policy->max); 1196 1197 cpudata->policy = policy->policy; 1198 1199 amd_pstate_epp_init(policy->cpu); 1200 1201 return 0; 1202 } 1203 1204 static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) 1205 { 1206 struct cppc_perf_ctrls perf_ctrls; 1207 u64 value, max_perf; 1208 int ret; 1209 1210 ret = amd_pstate_enable(true); 1211 if (ret) 1212 pr_err("failed to enable amd pstate during resume, return %d\n", ret); 1213 1214 value = READ_ONCE(cpudata->cppc_req_cached); 1215 max_perf = READ_ONCE(cpudata->highest_perf); 1216 1217 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1218 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1219 } else { 1220 perf_ctrls.max_perf = max_perf; 1221 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); 1222 cppc_set_perf(cpudata->cpu, &perf_ctrls); 1223 } 1224 } 1225 1226 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) 1227 { 1228 struct amd_cpudata *cpudata = policy->driver_data; 1229 1230 pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); 1231 1232 if (cppc_state == AMD_PSTATE_ACTIVE) { 1233 amd_pstate_epp_reenable(cpudata); 1234 cpudata->suspended = false; 1235 } 1236 1237 return 0; 1238 } 1239 1240 static void amd_pstate_epp_offline(struct cpufreq_policy *policy) 1241 { 1242 struct amd_cpudata *cpudata = policy->driver_data; 1243 struct cppc_perf_ctrls perf_ctrls; 1244 int min_perf; 1245 u64 value; 1246 1247 min_perf = READ_ONCE(cpudata->lowest_perf); 1248 value = READ_ONCE(cpudata->cppc_req_cached); 1249 1250 mutex_lock(&amd_pstate_limits_lock); 1251 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1252 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; 1253 1254 /* Set max perf same as min perf */ 1255 value &= ~AMD_CPPC_MAX_PERF(~0L); 1256 value |= AMD_CPPC_MAX_PERF(min_perf); 1257 value &= ~AMD_CPPC_MIN_PERF(~0L); 1258 value |= AMD_CPPC_MIN_PERF(min_perf); 1259 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1260 } else { 1261 perf_ctrls.desired_perf = 0; 1262 perf_ctrls.max_perf = min_perf; 1263 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); 1264 cppc_set_perf(cpudata->cpu, &perf_ctrls); 1265 } 1266 mutex_unlock(&amd_pstate_limits_lock); 1267 } 1268 1269 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) 1270 { 1271 struct amd_cpudata *cpudata = policy->driver_data; 1272 1273 pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); 1274 1275 if (cpudata->suspended) 1276 return 0; 1277 1278 if (cppc_state == AMD_PSTATE_ACTIVE) 1279 amd_pstate_epp_offline(policy); 1280 1281 return 0; 1282 } 1283 1284 static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) 1285 { 1286 cpufreq_verify_within_cpu_limits(policy); 1287 pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); 1288 return 0; 1289 } 1290 1291 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) 1292 { 1293 struct amd_cpudata *cpudata = policy->driver_data; 1294 int ret; 1295 1296 /* avoid suspending when EPP is not enabled */ 1297 if (cppc_state != AMD_PSTATE_ACTIVE) 1298 return 0; 1299 1300 /* set this flag to avoid setting core offline*/ 1301 cpudata->suspended = true; 1302 1303 /* disable CPPC in lowlevel firmware */ 1304 ret = amd_pstate_enable(false); 1305 if (ret) 1306 pr_err("failed to suspend, return %d\n", ret); 1307 1308 return 0; 1309 } 1310 1311 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 1312 { 1313 struct amd_cpudata *cpudata = policy->driver_data; 1314 1315 if (cpudata->suspended) { 1316 mutex_lock(&amd_pstate_limits_lock); 1317 1318 /* enable amd pstate from suspend state*/ 1319 amd_pstate_epp_reenable(cpudata); 1320 1321 mutex_unlock(&amd_pstate_limits_lock); 1322 1323 cpudata->suspended = false; 1324 } 1325 1326 return 0; 1327 } 1328 1329 static struct cpufreq_driver amd_pstate_driver = { 1330 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 1331 .verify = amd_pstate_verify, 1332 .target = amd_pstate_target, 1333 .fast_switch = amd_pstate_fast_switch, 1334 .init = amd_pstate_cpu_init, 1335 .exit = amd_pstate_cpu_exit, 1336 .suspend = amd_pstate_cpu_suspend, 1337 .resume = amd_pstate_cpu_resume, 1338 .set_boost = amd_pstate_set_boost, 1339 .name = "amd-pstate", 1340 .attr = amd_pstate_attr, 1341 }; 1342 1343 static struct cpufreq_driver amd_pstate_epp_driver = { 1344 .flags = CPUFREQ_CONST_LOOPS, 1345 .verify = amd_pstate_epp_verify_policy, 1346 .setpolicy = amd_pstate_epp_set_policy, 1347 .init = amd_pstate_epp_cpu_init, 1348 .exit = amd_pstate_epp_cpu_exit, 1349 .offline = amd_pstate_epp_cpu_offline, 1350 .online = amd_pstate_epp_cpu_online, 1351 .suspend = amd_pstate_epp_suspend, 1352 .resume = amd_pstate_epp_resume, 1353 .name = "amd_pstate_epp", 1354 .attr = amd_pstate_epp_attr, 1355 }; 1356 1357 static int __init amd_pstate_init(void) 1358 { 1359 struct device *dev_root; 1360 int ret; 1361 1362 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 1363 return -ENODEV; 1364 /* 1365 * by default the pstate driver is disabled to load 1366 * enable the amd_pstate passive mode driver explicitly 1367 * with amd_pstate=passive or other modes in kernel command line 1368 */ 1369 if (cppc_state == AMD_PSTATE_DISABLE) { 1370 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 1371 return -ENODEV; 1372 } 1373 1374 if (!acpi_cpc_valid()) { 1375 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 1376 return -ENODEV; 1377 } 1378 1379 /* don't keep reloading if cpufreq_driver exists */ 1380 if (cpufreq_get_current_driver()) 1381 return -EEXIST; 1382 1383 /* capability check */ 1384 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1385 pr_debug("AMD CPPC MSR based functionality is supported\n"); 1386 if (cppc_state != AMD_PSTATE_ACTIVE) 1387 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1388 } else { 1389 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 1390 static_call_update(amd_pstate_enable, cppc_enable); 1391 static_call_update(amd_pstate_init_perf, cppc_init_perf); 1392 static_call_update(amd_pstate_update_perf, cppc_update_perf); 1393 } 1394 1395 /* enable amd pstate feature */ 1396 ret = amd_pstate_enable(true); 1397 if (ret) { 1398 pr_err("failed to enable with return %d\n", ret); 1399 return ret; 1400 } 1401 1402 ret = cpufreq_register_driver(current_pstate_driver); 1403 if (ret) 1404 pr_err("failed to register with return %d\n", ret); 1405 1406 dev_root = bus_get_dev_root(&cpu_subsys); 1407 if (dev_root) { 1408 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); 1409 put_device(dev_root); 1410 if (ret) { 1411 pr_err("sysfs attribute export failed with error %d.\n", ret); 1412 goto global_attr_free; 1413 } 1414 } 1415 1416 return ret; 1417 1418 global_attr_free: 1419 cpufreq_unregister_driver(current_pstate_driver); 1420 return ret; 1421 } 1422 device_initcall(amd_pstate_init); 1423 1424 static int __init amd_pstate_param(char *str) 1425 { 1426 size_t size; 1427 int mode_idx; 1428 1429 if (!str) 1430 return -EINVAL; 1431 1432 size = strlen(str); 1433 mode_idx = get_mode_idx_from_str(str, size); 1434 1435 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1436 cppc_state = mode_idx; 1437 if (cppc_state == AMD_PSTATE_DISABLE) 1438 pr_info("driver is explicitly disabled\n"); 1439 1440 if (cppc_state == AMD_PSTATE_ACTIVE) 1441 current_pstate_driver = &amd_pstate_epp_driver; 1442 1443 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1444 current_pstate_driver = &amd_pstate_driver; 1445 1446 return 0; 1447 } 1448 1449 return -EINVAL; 1450 } 1451 early_param("amd_pstate", amd_pstate_param); 1452 1453 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 1454 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 1455