1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/kernel.h> 26 #include <linux/module.h> 27 #include <linux/init.h> 28 #include <linux/smp.h> 29 #include <linux/sched.h> 30 #include <linux/cpufreq.h> 31 #include <linux/compiler.h> 32 #include <linux/dmi.h> 33 #include <linux/slab.h> 34 #include <linux/acpi.h> 35 #include <linux/io.h> 36 #include <linux/delay.h> 37 #include <linux/uaccess.h> 38 #include <linux/static_call.h> 39 #include <linux/amd-pstate.h> 40 41 #include <acpi/processor.h> 42 #include <acpi/cppc_acpi.h> 43 44 #include <asm/msr.h> 45 #include <asm/processor.h> 46 #include <asm/cpufeature.h> 47 #include <asm/cpu_device_id.h> 48 #include "amd-pstate-trace.h" 49 50 #define AMD_PSTATE_TRANSITION_LATENCY 20000 51 #define AMD_PSTATE_TRANSITION_DELAY 1000 52 53 /* 54 * TODO: We need more time to fine tune processors with shared memory solution 55 * with community together. 56 * 57 * There are some performance drops on the CPU benchmarks which reports from 58 * Suse. We are co-working with them to fine tune the shared memory solution. So 59 * we disable it by default to go acpi-cpufreq on these processors and add a 60 * module parameter to be able to enable it manually for debugging. 61 */ 62 static struct cpufreq_driver *current_pstate_driver; 63 static struct cpufreq_driver amd_pstate_driver; 64 static struct cpufreq_driver amd_pstate_epp_driver; 65 static int cppc_state = AMD_PSTATE_DISABLE; 66 67 /* 68 * AMD Energy Preference Performance (EPP) 69 * The EPP is used in the CCLK DPM controller to drive 70 * the frequency that a core is going to operate during 71 * short periods of activity. EPP values will be utilized for 72 * different OS profiles (balanced, performance, power savings) 73 * display strings corresponding to EPP index in the 74 * energy_perf_strings[] 75 * index String 76 *------------------------------------- 77 * 0 default 78 * 1 performance 79 * 2 balance_performance 80 * 3 balance_power 81 * 4 power 82 */ 83 enum energy_perf_value_index { 84 EPP_INDEX_DEFAULT = 0, 85 EPP_INDEX_PERFORMANCE, 86 EPP_INDEX_BALANCE_PERFORMANCE, 87 EPP_INDEX_BALANCE_POWERSAVE, 88 EPP_INDEX_POWERSAVE, 89 }; 90 91 static const char * const energy_perf_strings[] = { 92 [EPP_INDEX_DEFAULT] = "default", 93 [EPP_INDEX_PERFORMANCE] = "performance", 94 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 95 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 96 [EPP_INDEX_POWERSAVE] = "power", 97 NULL 98 }; 99 100 static unsigned int epp_values[] = { 101 [EPP_INDEX_DEFAULT] = 0, 102 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 103 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 104 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 105 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 106 }; 107 108 typedef int (*cppc_mode_transition_fn)(int); 109 110 static inline int get_mode_idx_from_str(const char *str, size_t size) 111 { 112 int i; 113 114 for (i=0; i < AMD_PSTATE_MAX; i++) { 115 if (!strncmp(str, amd_pstate_mode_string[i], size)) 116 return i; 117 } 118 return -EINVAL; 119 } 120 121 static DEFINE_MUTEX(amd_pstate_limits_lock); 122 static DEFINE_MUTEX(amd_pstate_driver_lock); 123 124 static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) 125 { 126 u64 epp; 127 int ret; 128 129 if (boot_cpu_has(X86_FEATURE_CPPC)) { 130 if (!cppc_req_cached) { 131 epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 132 &cppc_req_cached); 133 if (epp) 134 return epp; 135 } 136 epp = (cppc_req_cached >> 24) & 0xFF; 137 } else { 138 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 139 if (ret < 0) { 140 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 141 return -EIO; 142 } 143 } 144 145 return (s16)(epp & 0xff); 146 } 147 148 static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) 149 { 150 s16 epp; 151 int index = -EINVAL; 152 153 epp = amd_pstate_get_epp(cpudata, 0); 154 if (epp < 0) 155 return epp; 156 157 switch (epp) { 158 case AMD_CPPC_EPP_PERFORMANCE: 159 index = EPP_INDEX_PERFORMANCE; 160 break; 161 case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 162 index = EPP_INDEX_BALANCE_PERFORMANCE; 163 break; 164 case AMD_CPPC_EPP_BALANCE_POWERSAVE: 165 index = EPP_INDEX_BALANCE_POWERSAVE; 166 break; 167 case AMD_CPPC_EPP_POWERSAVE: 168 index = EPP_INDEX_POWERSAVE; 169 break; 170 default: 171 break; 172 } 173 174 return index; 175 } 176 177 static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) 178 { 179 int ret; 180 struct cppc_perf_ctrls perf_ctrls; 181 182 if (boot_cpu_has(X86_FEATURE_CPPC)) { 183 u64 value = READ_ONCE(cpudata->cppc_req_cached); 184 185 value &= ~GENMASK_ULL(31, 24); 186 value |= (u64)epp << 24; 187 WRITE_ONCE(cpudata->cppc_req_cached, value); 188 189 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 190 if (!ret) 191 cpudata->epp_cached = epp; 192 } else { 193 perf_ctrls.energy_perf = epp; 194 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 195 if (ret) { 196 pr_debug("failed to set energy perf value (%d)\n", ret); 197 return ret; 198 } 199 cpudata->epp_cached = epp; 200 } 201 202 return ret; 203 } 204 205 static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, 206 int pref_index) 207 { 208 int epp = -EINVAL; 209 int ret; 210 211 if (!pref_index) { 212 pr_debug("EPP pref_index is invalid\n"); 213 return -EINVAL; 214 } 215 216 if (epp == -EINVAL) 217 epp = epp_values[pref_index]; 218 219 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 220 pr_debug("EPP cannot be set under performance policy\n"); 221 return -EBUSY; 222 } 223 224 ret = amd_pstate_set_epp(cpudata, epp); 225 226 return ret; 227 } 228 229 static inline int pstate_enable(bool enable) 230 { 231 return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); 232 } 233 234 static int cppc_enable(bool enable) 235 { 236 int cpu, ret = 0; 237 struct cppc_perf_ctrls perf_ctrls; 238 239 for_each_present_cpu(cpu) { 240 ret = cppc_set_enable(cpu, enable); 241 if (ret) 242 return ret; 243 244 /* Enable autonomous mode for EPP */ 245 if (cppc_state == AMD_PSTATE_ACTIVE) { 246 /* Set desired perf as zero to allow EPP firmware control */ 247 perf_ctrls.desired_perf = 0; 248 ret = cppc_set_perf(cpu, &perf_ctrls); 249 if (ret) 250 return ret; 251 } 252 } 253 254 return ret; 255 } 256 257 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); 258 259 static inline int amd_pstate_enable(bool enable) 260 { 261 return static_call(amd_pstate_enable)(enable); 262 } 263 264 static int pstate_init_perf(struct amd_cpudata *cpudata) 265 { 266 u64 cap1; 267 u32 highest_perf; 268 269 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 270 &cap1); 271 if (ret) 272 return ret; 273 274 /* 275 * TODO: Introduce AMD specific power feature. 276 * 277 * CPPC entry doesn't indicate the highest performance in some ASICs. 278 */ 279 highest_perf = amd_get_highest_perf(); 280 if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) 281 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); 282 283 WRITE_ONCE(cpudata->highest_perf, highest_perf); 284 285 WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); 286 WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); 287 WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); 288 289 return 0; 290 } 291 292 static int cppc_init_perf(struct amd_cpudata *cpudata) 293 { 294 struct cppc_perf_caps cppc_perf; 295 u32 highest_perf; 296 297 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 298 if (ret) 299 return ret; 300 301 highest_perf = amd_get_highest_perf(); 302 if (highest_perf > cppc_perf.highest_perf) 303 highest_perf = cppc_perf.highest_perf; 304 305 WRITE_ONCE(cpudata->highest_perf, highest_perf); 306 307 WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); 308 WRITE_ONCE(cpudata->lowest_nonlinear_perf, 309 cppc_perf.lowest_nonlinear_perf); 310 WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); 311 312 if (cppc_state == AMD_PSTATE_ACTIVE) 313 return 0; 314 315 ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf); 316 if (ret) { 317 pr_warn("failed to get auto_sel, ret: %d\n", ret); 318 return 0; 319 } 320 321 ret = cppc_set_auto_sel(cpudata->cpu, 322 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 323 324 if (ret) 325 pr_warn("failed to set auto_sel, ret: %d\n", ret); 326 327 return ret; 328 } 329 330 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); 331 332 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 333 { 334 return static_call(amd_pstate_init_perf)(cpudata); 335 } 336 337 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, 338 u32 des_perf, u32 max_perf, bool fast_switch) 339 { 340 if (fast_switch) 341 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); 342 else 343 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 344 READ_ONCE(cpudata->cppc_req_cached)); 345 } 346 347 static void cppc_update_perf(struct amd_cpudata *cpudata, 348 u32 min_perf, u32 des_perf, 349 u32 max_perf, bool fast_switch) 350 { 351 struct cppc_perf_ctrls perf_ctrls; 352 353 perf_ctrls.max_perf = max_perf; 354 perf_ctrls.min_perf = min_perf; 355 perf_ctrls.desired_perf = des_perf; 356 357 cppc_set_perf(cpudata->cpu, &perf_ctrls); 358 } 359 360 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); 361 362 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, 363 u32 min_perf, u32 des_perf, 364 u32 max_perf, bool fast_switch) 365 { 366 static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, 367 max_perf, fast_switch); 368 } 369 370 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 371 { 372 u64 aperf, mperf, tsc; 373 unsigned long flags; 374 375 local_irq_save(flags); 376 rdmsrl(MSR_IA32_APERF, aperf); 377 rdmsrl(MSR_IA32_MPERF, mperf); 378 tsc = rdtsc(); 379 380 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 381 local_irq_restore(flags); 382 return false; 383 } 384 385 local_irq_restore(flags); 386 387 cpudata->cur.aperf = aperf; 388 cpudata->cur.mperf = mperf; 389 cpudata->cur.tsc = tsc; 390 cpudata->cur.aperf -= cpudata->prev.aperf; 391 cpudata->cur.mperf -= cpudata->prev.mperf; 392 cpudata->cur.tsc -= cpudata->prev.tsc; 393 394 cpudata->prev.aperf = aperf; 395 cpudata->prev.mperf = mperf; 396 cpudata->prev.tsc = tsc; 397 398 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 399 400 return true; 401 } 402 403 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, 404 u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags) 405 { 406 u64 prev = READ_ONCE(cpudata->cppc_req_cached); 407 u64 value = prev; 408 409 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 410 411 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 412 min_perf = des_perf; 413 des_perf = 0; 414 } 415 416 value &= ~AMD_CPPC_MIN_PERF(~0L); 417 value |= AMD_CPPC_MIN_PERF(min_perf); 418 419 value &= ~AMD_CPPC_DES_PERF(~0L); 420 value |= AMD_CPPC_DES_PERF(des_perf); 421 422 value &= ~AMD_CPPC_MAX_PERF(~0L); 423 value |= AMD_CPPC_MAX_PERF(max_perf); 424 425 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 426 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 427 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 428 cpudata->cpu, (value != prev), fast_switch); 429 } 430 431 if (value == prev) 432 return; 433 434 WRITE_ONCE(cpudata->cppc_req_cached, value); 435 436 amd_pstate_update_perf(cpudata, min_perf, des_perf, 437 max_perf, fast_switch); 438 } 439 440 static int amd_pstate_verify(struct cpufreq_policy_data *policy) 441 { 442 cpufreq_verify_within_cpu_limits(policy); 443 444 return 0; 445 } 446 447 static int amd_pstate_target(struct cpufreq_policy *policy, 448 unsigned int target_freq, 449 unsigned int relation) 450 { 451 struct cpufreq_freqs freqs; 452 struct amd_cpudata *cpudata = policy->driver_data; 453 unsigned long max_perf, min_perf, des_perf, cap_perf; 454 455 if (!cpudata->max_freq) 456 return -ENODEV; 457 458 cap_perf = READ_ONCE(cpudata->highest_perf); 459 min_perf = READ_ONCE(cpudata->lowest_perf); 460 max_perf = cap_perf; 461 462 freqs.old = policy->cur; 463 freqs.new = target_freq; 464 465 des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, 466 cpudata->max_freq); 467 468 cpufreq_freq_transition_begin(policy, &freqs); 469 amd_pstate_update(cpudata, min_perf, des_perf, 470 max_perf, false, policy->governor->flags); 471 cpufreq_freq_transition_end(policy, &freqs, false); 472 473 return 0; 474 } 475 476 static void amd_pstate_adjust_perf(unsigned int cpu, 477 unsigned long _min_perf, 478 unsigned long target_perf, 479 unsigned long capacity) 480 { 481 unsigned long max_perf, min_perf, des_perf, 482 cap_perf, lowest_nonlinear_perf; 483 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 484 struct amd_cpudata *cpudata = policy->driver_data; 485 486 cap_perf = READ_ONCE(cpudata->highest_perf); 487 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); 488 489 des_perf = cap_perf; 490 if (target_perf < capacity) 491 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 492 493 min_perf = READ_ONCE(cpudata->highest_perf); 494 if (_min_perf < capacity) 495 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 496 497 if (min_perf < lowest_nonlinear_perf) 498 min_perf = lowest_nonlinear_perf; 499 500 max_perf = cap_perf; 501 if (max_perf < min_perf) 502 max_perf = min_perf; 503 504 amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, 505 policy->governor->flags); 506 cpufreq_cpu_put(policy); 507 } 508 509 static int amd_get_min_freq(struct amd_cpudata *cpudata) 510 { 511 struct cppc_perf_caps cppc_perf; 512 513 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 514 if (ret) 515 return ret; 516 517 /* Switch to khz */ 518 return cppc_perf.lowest_freq * 1000; 519 } 520 521 static int amd_get_max_freq(struct amd_cpudata *cpudata) 522 { 523 struct cppc_perf_caps cppc_perf; 524 u32 max_perf, max_freq, nominal_freq, nominal_perf; 525 u64 boost_ratio; 526 527 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 528 if (ret) 529 return ret; 530 531 nominal_freq = cppc_perf.nominal_freq; 532 nominal_perf = READ_ONCE(cpudata->nominal_perf); 533 max_perf = READ_ONCE(cpudata->highest_perf); 534 535 boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT, 536 nominal_perf); 537 538 max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT; 539 540 /* Switch to khz */ 541 return max_freq * 1000; 542 } 543 544 static int amd_get_nominal_freq(struct amd_cpudata *cpudata) 545 { 546 struct cppc_perf_caps cppc_perf; 547 548 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 549 if (ret) 550 return ret; 551 552 /* Switch to khz */ 553 return cppc_perf.nominal_freq * 1000; 554 } 555 556 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) 557 { 558 struct cppc_perf_caps cppc_perf; 559 u32 lowest_nonlinear_freq, lowest_nonlinear_perf, 560 nominal_freq, nominal_perf; 561 u64 lowest_nonlinear_ratio; 562 563 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 564 if (ret) 565 return ret; 566 567 nominal_freq = cppc_perf.nominal_freq; 568 nominal_perf = READ_ONCE(cpudata->nominal_perf); 569 570 lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 571 572 lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, 573 nominal_perf); 574 575 lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT; 576 577 /* Switch to khz */ 578 return lowest_nonlinear_freq * 1000; 579 } 580 581 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 582 { 583 struct amd_cpudata *cpudata = policy->driver_data; 584 int ret; 585 586 if (!cpudata->boost_supported) { 587 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 588 return -EINVAL; 589 } 590 591 if (state) 592 policy->cpuinfo.max_freq = cpudata->max_freq; 593 else 594 policy->cpuinfo.max_freq = cpudata->nominal_freq; 595 596 policy->max = policy->cpuinfo.max_freq; 597 598 ret = freq_qos_update_request(&cpudata->req[1], 599 policy->cpuinfo.max_freq); 600 if (ret < 0) 601 return ret; 602 603 return 0; 604 } 605 606 static void amd_pstate_boost_init(struct amd_cpudata *cpudata) 607 { 608 u32 highest_perf, nominal_perf; 609 610 highest_perf = READ_ONCE(cpudata->highest_perf); 611 nominal_perf = READ_ONCE(cpudata->nominal_perf); 612 613 if (highest_perf <= nominal_perf) 614 return; 615 616 cpudata->boost_supported = true; 617 current_pstate_driver->boost_enabled = true; 618 } 619 620 static void amd_perf_ctl_reset(unsigned int cpu) 621 { 622 wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 623 } 624 625 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 626 { 627 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 628 struct device *dev; 629 struct amd_cpudata *cpudata; 630 631 /* 632 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 633 * which is ideal for initialization process. 634 */ 635 amd_perf_ctl_reset(policy->cpu); 636 dev = get_cpu_device(policy->cpu); 637 if (!dev) 638 return -ENODEV; 639 640 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 641 if (!cpudata) 642 return -ENOMEM; 643 644 cpudata->cpu = policy->cpu; 645 646 ret = amd_pstate_init_perf(cpudata); 647 if (ret) 648 goto free_cpudata1; 649 650 min_freq = amd_get_min_freq(cpudata); 651 max_freq = amd_get_max_freq(cpudata); 652 nominal_freq = amd_get_nominal_freq(cpudata); 653 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 654 655 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 656 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 657 min_freq, max_freq); 658 ret = -EINVAL; 659 goto free_cpudata1; 660 } 661 662 policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; 663 policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; 664 665 policy->min = min_freq; 666 policy->max = max_freq; 667 668 policy->cpuinfo.min_freq = min_freq; 669 policy->cpuinfo.max_freq = max_freq; 670 671 /* It will be updated by governor */ 672 policy->cur = policy->cpuinfo.min_freq; 673 674 if (boot_cpu_has(X86_FEATURE_CPPC)) 675 policy->fast_switch_possible = true; 676 677 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 678 FREQ_QOS_MIN, policy->cpuinfo.min_freq); 679 if (ret < 0) { 680 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 681 goto free_cpudata1; 682 } 683 684 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 685 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 686 if (ret < 0) { 687 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 688 goto free_cpudata2; 689 } 690 691 /* Initial processor data capability frequencies */ 692 cpudata->max_freq = max_freq; 693 cpudata->min_freq = min_freq; 694 cpudata->nominal_freq = nominal_freq; 695 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 696 697 policy->driver_data = cpudata; 698 699 amd_pstate_boost_init(cpudata); 700 if (!current_pstate_driver->adjust_perf) 701 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 702 703 return 0; 704 705 free_cpudata2: 706 freq_qos_remove_request(&cpudata->req[0]); 707 free_cpudata1: 708 kfree(cpudata); 709 return ret; 710 } 711 712 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) 713 { 714 struct amd_cpudata *cpudata = policy->driver_data; 715 716 freq_qos_remove_request(&cpudata->req[1]); 717 freq_qos_remove_request(&cpudata->req[0]); 718 kfree(cpudata); 719 720 return 0; 721 } 722 723 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) 724 { 725 int ret; 726 727 ret = amd_pstate_enable(true); 728 if (ret) 729 pr_err("failed to enable amd-pstate during resume, return %d\n", ret); 730 731 return ret; 732 } 733 734 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) 735 { 736 int ret; 737 738 ret = amd_pstate_enable(false); 739 if (ret) 740 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); 741 742 return ret; 743 } 744 745 /* Sysfs attributes */ 746 747 /* 748 * This frequency is to indicate the maximum hardware frequency. 749 * If boost is not active but supported, the frequency will be larger than the 750 * one in cpuinfo. 751 */ 752 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 753 char *buf) 754 { 755 int max_freq; 756 struct amd_cpudata *cpudata = policy->driver_data; 757 758 max_freq = amd_get_max_freq(cpudata); 759 if (max_freq < 0) 760 return max_freq; 761 762 return sysfs_emit(buf, "%u\n", max_freq); 763 } 764 765 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 766 char *buf) 767 { 768 int freq; 769 struct amd_cpudata *cpudata = policy->driver_data; 770 771 freq = amd_get_lowest_nonlinear_freq(cpudata); 772 if (freq < 0) 773 return freq; 774 775 return sysfs_emit(buf, "%u\n", freq); 776 } 777 778 /* 779 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 780 * need to expose it to sysfs. 781 */ 782 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 783 char *buf) 784 { 785 u32 perf; 786 struct amd_cpudata *cpudata = policy->driver_data; 787 788 perf = READ_ONCE(cpudata->highest_perf); 789 790 return sysfs_emit(buf, "%u\n", perf); 791 } 792 793 static ssize_t show_energy_performance_available_preferences( 794 struct cpufreq_policy *policy, char *buf) 795 { 796 int i = 0; 797 int offset = 0; 798 799 while (energy_perf_strings[i] != NULL) 800 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); 801 802 sysfs_emit_at(buf, offset, "\n"); 803 804 return offset; 805 } 806 807 static ssize_t store_energy_performance_preference( 808 struct cpufreq_policy *policy, const char *buf, size_t count) 809 { 810 struct amd_cpudata *cpudata = policy->driver_data; 811 char str_preference[21]; 812 ssize_t ret; 813 814 ret = sscanf(buf, "%20s", str_preference); 815 if (ret != 1) 816 return -EINVAL; 817 818 ret = match_string(energy_perf_strings, -1, str_preference); 819 if (ret < 0) 820 return -EINVAL; 821 822 mutex_lock(&amd_pstate_limits_lock); 823 ret = amd_pstate_set_energy_pref_index(cpudata, ret); 824 mutex_unlock(&amd_pstate_limits_lock); 825 826 return ret ?: count; 827 } 828 829 static ssize_t show_energy_performance_preference( 830 struct cpufreq_policy *policy, char *buf) 831 { 832 struct amd_cpudata *cpudata = policy->driver_data; 833 int preference; 834 835 preference = amd_pstate_get_energy_pref_index(cpudata); 836 if (preference < 0) 837 return preference; 838 839 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 840 } 841 842 static void amd_pstate_driver_cleanup(void) 843 { 844 amd_pstate_enable(false); 845 cppc_state = AMD_PSTATE_DISABLE; 846 current_pstate_driver = NULL; 847 } 848 849 static int amd_pstate_register_driver(int mode) 850 { 851 int ret; 852 853 if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED) 854 current_pstate_driver = &amd_pstate_driver; 855 else if (mode == AMD_PSTATE_ACTIVE) 856 current_pstate_driver = &amd_pstate_epp_driver; 857 else 858 return -EINVAL; 859 860 cppc_state = mode; 861 ret = cpufreq_register_driver(current_pstate_driver); 862 if (ret) { 863 amd_pstate_driver_cleanup(); 864 return ret; 865 } 866 return 0; 867 } 868 869 static int amd_pstate_unregister_driver(int dummy) 870 { 871 cpufreq_unregister_driver(current_pstate_driver); 872 amd_pstate_driver_cleanup(); 873 return 0; 874 } 875 876 static int amd_pstate_change_mode_without_dvr_change(int mode) 877 { 878 int cpu = 0; 879 880 cppc_state = mode; 881 882 if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 883 return 0; 884 885 for_each_present_cpu(cpu) { 886 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 887 } 888 889 return 0; 890 } 891 892 static int amd_pstate_change_driver_mode(int mode) 893 { 894 int ret; 895 896 ret = amd_pstate_unregister_driver(0); 897 if (ret) 898 return ret; 899 900 ret = amd_pstate_register_driver(mode); 901 if (ret) 902 return ret; 903 904 return 0; 905 } 906 907 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 908 [AMD_PSTATE_DISABLE] = { 909 [AMD_PSTATE_DISABLE] = NULL, 910 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 911 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 912 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 913 }, 914 [AMD_PSTATE_PASSIVE] = { 915 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 916 [AMD_PSTATE_PASSIVE] = NULL, 917 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 918 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 919 }, 920 [AMD_PSTATE_ACTIVE] = { 921 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 922 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 923 [AMD_PSTATE_ACTIVE] = NULL, 924 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 925 }, 926 [AMD_PSTATE_GUIDED] = { 927 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 928 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 929 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 930 [AMD_PSTATE_GUIDED] = NULL, 931 }, 932 }; 933 934 static ssize_t amd_pstate_show_status(char *buf) 935 { 936 if (!current_pstate_driver) 937 return sysfs_emit(buf, "disable\n"); 938 939 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 940 } 941 942 static int amd_pstate_update_status(const char *buf, size_t size) 943 { 944 int mode_idx; 945 946 if (size > strlen("passive") || size < strlen("active")) 947 return -EINVAL; 948 949 mode_idx = get_mode_idx_from_str(buf, size); 950 951 if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX) 952 return -EINVAL; 953 954 if (mode_state_machine[cppc_state][mode_idx]) 955 return mode_state_machine[cppc_state][mode_idx](mode_idx); 956 957 return 0; 958 } 959 960 static ssize_t show_status(struct kobject *kobj, 961 struct kobj_attribute *attr, char *buf) 962 { 963 ssize_t ret; 964 965 mutex_lock(&amd_pstate_driver_lock); 966 ret = amd_pstate_show_status(buf); 967 mutex_unlock(&amd_pstate_driver_lock); 968 969 return ret; 970 } 971 972 static ssize_t store_status(struct kobject *a, struct kobj_attribute *b, 973 const char *buf, size_t count) 974 { 975 char *p = memchr(buf, '\n', count); 976 int ret; 977 978 mutex_lock(&amd_pstate_driver_lock); 979 ret = amd_pstate_update_status(buf, p ? p - buf : count); 980 mutex_unlock(&amd_pstate_driver_lock); 981 982 return ret < 0 ? ret : count; 983 } 984 985 cpufreq_freq_attr_ro(amd_pstate_max_freq); 986 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 987 988 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 989 cpufreq_freq_attr_rw(energy_performance_preference); 990 cpufreq_freq_attr_ro(energy_performance_available_preferences); 991 define_one_global_rw(status); 992 993 static struct freq_attr *amd_pstate_attr[] = { 994 &amd_pstate_max_freq, 995 &amd_pstate_lowest_nonlinear_freq, 996 &amd_pstate_highest_perf, 997 NULL, 998 }; 999 1000 static struct freq_attr *amd_pstate_epp_attr[] = { 1001 &amd_pstate_max_freq, 1002 &amd_pstate_lowest_nonlinear_freq, 1003 &amd_pstate_highest_perf, 1004 &energy_performance_preference, 1005 &energy_performance_available_preferences, 1006 NULL, 1007 }; 1008 1009 static struct attribute *pstate_global_attributes[] = { 1010 &status.attr, 1011 NULL 1012 }; 1013 1014 static const struct attribute_group amd_pstate_global_attr_group = { 1015 .name = "amd_pstate", 1016 .attrs = pstate_global_attributes, 1017 }; 1018 1019 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1020 { 1021 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 1022 struct amd_cpudata *cpudata; 1023 struct device *dev; 1024 u64 value; 1025 1026 /* 1027 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1028 * which is ideal for initialization process. 1029 */ 1030 amd_perf_ctl_reset(policy->cpu); 1031 dev = get_cpu_device(policy->cpu); 1032 if (!dev) 1033 return -ENODEV; 1034 1035 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 1036 if (!cpudata) 1037 return -ENOMEM; 1038 1039 cpudata->cpu = policy->cpu; 1040 cpudata->epp_policy = 0; 1041 1042 ret = amd_pstate_init_perf(cpudata); 1043 if (ret) 1044 goto free_cpudata1; 1045 1046 min_freq = amd_get_min_freq(cpudata); 1047 max_freq = amd_get_max_freq(cpudata); 1048 nominal_freq = amd_get_nominal_freq(cpudata); 1049 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 1050 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 1051 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 1052 min_freq, max_freq); 1053 ret = -EINVAL; 1054 goto free_cpudata1; 1055 } 1056 1057 policy->cpuinfo.min_freq = min_freq; 1058 policy->cpuinfo.max_freq = max_freq; 1059 /* It will be updated by governor */ 1060 policy->cur = policy->cpuinfo.min_freq; 1061 1062 /* Initial processor data capability frequencies */ 1063 cpudata->max_freq = max_freq; 1064 cpudata->min_freq = min_freq; 1065 cpudata->nominal_freq = nominal_freq; 1066 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 1067 1068 policy->driver_data = cpudata; 1069 1070 cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); 1071 1072 policy->min = policy->cpuinfo.min_freq; 1073 policy->max = policy->cpuinfo.max_freq; 1074 1075 /* 1076 * Set the policy to powersave to provide a valid fallback value in case 1077 * the default cpufreq governor is neither powersave nor performance. 1078 */ 1079 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1080 1081 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1082 policy->fast_switch_possible = true; 1083 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 1084 if (ret) 1085 return ret; 1086 WRITE_ONCE(cpudata->cppc_req_cached, value); 1087 1088 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); 1089 if (ret) 1090 return ret; 1091 WRITE_ONCE(cpudata->cppc_cap1_cached, value); 1092 } 1093 amd_pstate_boost_init(cpudata); 1094 1095 return 0; 1096 1097 free_cpudata1: 1098 kfree(cpudata); 1099 return ret; 1100 } 1101 1102 static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1103 { 1104 pr_debug("CPU %d exiting\n", policy->cpu); 1105 policy->fast_switch_possible = false; 1106 return 0; 1107 } 1108 1109 static void amd_pstate_epp_init(unsigned int cpu) 1110 { 1111 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1112 struct amd_cpudata *cpudata = policy->driver_data; 1113 u32 max_perf, min_perf; 1114 u64 value; 1115 s16 epp; 1116 1117 max_perf = READ_ONCE(cpudata->highest_perf); 1118 min_perf = READ_ONCE(cpudata->lowest_perf); 1119 1120 value = READ_ONCE(cpudata->cppc_req_cached); 1121 1122 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1123 min_perf = max_perf; 1124 1125 /* Initial min/max values for CPPC Performance Controls Register */ 1126 value &= ~AMD_CPPC_MIN_PERF(~0L); 1127 value |= AMD_CPPC_MIN_PERF(min_perf); 1128 1129 value &= ~AMD_CPPC_MAX_PERF(~0L); 1130 value |= AMD_CPPC_MAX_PERF(max_perf); 1131 1132 /* CPPC EPP feature require to set zero to the desire perf bit */ 1133 value &= ~AMD_CPPC_DES_PERF(~0L); 1134 value |= AMD_CPPC_DES_PERF(0); 1135 1136 if (cpudata->epp_policy == cpudata->policy) 1137 goto skip_epp; 1138 1139 cpudata->epp_policy = cpudata->policy; 1140 1141 /* Get BIOS pre-defined epp value */ 1142 epp = amd_pstate_get_epp(cpudata, value); 1143 if (epp < 0) { 1144 /** 1145 * This return value can only be negative for shared_memory 1146 * systems where EPP register read/write not supported. 1147 */ 1148 goto skip_epp; 1149 } 1150 1151 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1152 epp = 0; 1153 1154 /* Set initial EPP value */ 1155 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1156 value &= ~GENMASK_ULL(31, 24); 1157 value |= (u64)epp << 24; 1158 } 1159 1160 WRITE_ONCE(cpudata->cppc_req_cached, value); 1161 amd_pstate_set_epp(cpudata, epp); 1162 skip_epp: 1163 cpufreq_cpu_put(policy); 1164 } 1165 1166 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 1167 { 1168 struct amd_cpudata *cpudata = policy->driver_data; 1169 1170 if (!policy->cpuinfo.max_freq) 1171 return -ENODEV; 1172 1173 pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", 1174 policy->cpuinfo.max_freq, policy->max); 1175 1176 cpudata->policy = policy->policy; 1177 1178 amd_pstate_epp_init(policy->cpu); 1179 1180 return 0; 1181 } 1182 1183 static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) 1184 { 1185 struct cppc_perf_ctrls perf_ctrls; 1186 u64 value, max_perf; 1187 int ret; 1188 1189 ret = amd_pstate_enable(true); 1190 if (ret) 1191 pr_err("failed to enable amd pstate during resume, return %d\n", ret); 1192 1193 value = READ_ONCE(cpudata->cppc_req_cached); 1194 max_perf = READ_ONCE(cpudata->highest_perf); 1195 1196 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1197 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1198 } else { 1199 perf_ctrls.max_perf = max_perf; 1200 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); 1201 cppc_set_perf(cpudata->cpu, &perf_ctrls); 1202 } 1203 } 1204 1205 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) 1206 { 1207 struct amd_cpudata *cpudata = policy->driver_data; 1208 1209 pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); 1210 1211 if (cppc_state == AMD_PSTATE_ACTIVE) { 1212 amd_pstate_epp_reenable(cpudata); 1213 cpudata->suspended = false; 1214 } 1215 1216 return 0; 1217 } 1218 1219 static void amd_pstate_epp_offline(struct cpufreq_policy *policy) 1220 { 1221 struct amd_cpudata *cpudata = policy->driver_data; 1222 struct cppc_perf_ctrls perf_ctrls; 1223 int min_perf; 1224 u64 value; 1225 1226 min_perf = READ_ONCE(cpudata->lowest_perf); 1227 value = READ_ONCE(cpudata->cppc_req_cached); 1228 1229 mutex_lock(&amd_pstate_limits_lock); 1230 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1231 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; 1232 1233 /* Set max perf same as min perf */ 1234 value &= ~AMD_CPPC_MAX_PERF(~0L); 1235 value |= AMD_CPPC_MAX_PERF(min_perf); 1236 value &= ~AMD_CPPC_MIN_PERF(~0L); 1237 value |= AMD_CPPC_MIN_PERF(min_perf); 1238 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1239 } else { 1240 perf_ctrls.desired_perf = 0; 1241 perf_ctrls.max_perf = min_perf; 1242 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); 1243 cppc_set_perf(cpudata->cpu, &perf_ctrls); 1244 } 1245 mutex_unlock(&amd_pstate_limits_lock); 1246 } 1247 1248 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) 1249 { 1250 struct amd_cpudata *cpudata = policy->driver_data; 1251 1252 pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); 1253 1254 if (cpudata->suspended) 1255 return 0; 1256 1257 if (cppc_state == AMD_PSTATE_ACTIVE) 1258 amd_pstate_epp_offline(policy); 1259 1260 return 0; 1261 } 1262 1263 static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) 1264 { 1265 cpufreq_verify_within_cpu_limits(policy); 1266 pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); 1267 return 0; 1268 } 1269 1270 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) 1271 { 1272 struct amd_cpudata *cpudata = policy->driver_data; 1273 int ret; 1274 1275 /* avoid suspending when EPP is not enabled */ 1276 if (cppc_state != AMD_PSTATE_ACTIVE) 1277 return 0; 1278 1279 /* set this flag to avoid setting core offline*/ 1280 cpudata->suspended = true; 1281 1282 /* disable CPPC in lowlevel firmware */ 1283 ret = amd_pstate_enable(false); 1284 if (ret) 1285 pr_err("failed to suspend, return %d\n", ret); 1286 1287 return 0; 1288 } 1289 1290 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 1291 { 1292 struct amd_cpudata *cpudata = policy->driver_data; 1293 1294 if (cpudata->suspended) { 1295 mutex_lock(&amd_pstate_limits_lock); 1296 1297 /* enable amd pstate from suspend state*/ 1298 amd_pstate_epp_reenable(cpudata); 1299 1300 mutex_unlock(&amd_pstate_limits_lock); 1301 1302 cpudata->suspended = false; 1303 } 1304 1305 return 0; 1306 } 1307 1308 static struct cpufreq_driver amd_pstate_driver = { 1309 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 1310 .verify = amd_pstate_verify, 1311 .target = amd_pstate_target, 1312 .init = amd_pstate_cpu_init, 1313 .exit = amd_pstate_cpu_exit, 1314 .suspend = amd_pstate_cpu_suspend, 1315 .resume = amd_pstate_cpu_resume, 1316 .set_boost = amd_pstate_set_boost, 1317 .name = "amd-pstate", 1318 .attr = amd_pstate_attr, 1319 }; 1320 1321 static struct cpufreq_driver amd_pstate_epp_driver = { 1322 .flags = CPUFREQ_CONST_LOOPS, 1323 .verify = amd_pstate_epp_verify_policy, 1324 .setpolicy = amd_pstate_epp_set_policy, 1325 .init = amd_pstate_epp_cpu_init, 1326 .exit = amd_pstate_epp_cpu_exit, 1327 .offline = amd_pstate_epp_cpu_offline, 1328 .online = amd_pstate_epp_cpu_online, 1329 .suspend = amd_pstate_epp_suspend, 1330 .resume = amd_pstate_epp_resume, 1331 .name = "amd_pstate_epp", 1332 .attr = amd_pstate_epp_attr, 1333 }; 1334 1335 static int __init amd_pstate_init(void) 1336 { 1337 struct device *dev_root; 1338 int ret; 1339 1340 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 1341 return -ENODEV; 1342 /* 1343 * by default the pstate driver is disabled to load 1344 * enable the amd_pstate passive mode driver explicitly 1345 * with amd_pstate=passive or other modes in kernel command line 1346 */ 1347 if (cppc_state == AMD_PSTATE_DISABLE) { 1348 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 1349 return -ENODEV; 1350 } 1351 1352 if (!acpi_cpc_valid()) { 1353 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 1354 return -ENODEV; 1355 } 1356 1357 /* don't keep reloading if cpufreq_driver exists */ 1358 if (cpufreq_get_current_driver()) 1359 return -EEXIST; 1360 1361 /* capability check */ 1362 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1363 pr_debug("AMD CPPC MSR based functionality is supported\n"); 1364 if (cppc_state != AMD_PSTATE_ACTIVE) 1365 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1366 } else { 1367 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 1368 static_call_update(amd_pstate_enable, cppc_enable); 1369 static_call_update(amd_pstate_init_perf, cppc_init_perf); 1370 static_call_update(amd_pstate_update_perf, cppc_update_perf); 1371 } 1372 1373 /* enable amd pstate feature */ 1374 ret = amd_pstate_enable(true); 1375 if (ret) { 1376 pr_err("failed to enable with return %d\n", ret); 1377 return ret; 1378 } 1379 1380 ret = cpufreq_register_driver(current_pstate_driver); 1381 if (ret) 1382 pr_err("failed to register with return %d\n", ret); 1383 1384 dev_root = bus_get_dev_root(&cpu_subsys); 1385 if (dev_root) { 1386 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); 1387 put_device(dev_root); 1388 if (ret) { 1389 pr_err("sysfs attribute export failed with error %d.\n", ret); 1390 goto global_attr_free; 1391 } 1392 } 1393 1394 return ret; 1395 1396 global_attr_free: 1397 cpufreq_unregister_driver(current_pstate_driver); 1398 return ret; 1399 } 1400 device_initcall(amd_pstate_init); 1401 1402 static int __init amd_pstate_param(char *str) 1403 { 1404 size_t size; 1405 int mode_idx; 1406 1407 if (!str) 1408 return -EINVAL; 1409 1410 size = strlen(str); 1411 mode_idx = get_mode_idx_from_str(str, size); 1412 1413 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1414 cppc_state = mode_idx; 1415 if (cppc_state == AMD_PSTATE_DISABLE) 1416 pr_info("driver is explicitly disabled\n"); 1417 1418 if (cppc_state == AMD_PSTATE_ACTIVE) 1419 current_pstate_driver = &amd_pstate_epp_driver; 1420 1421 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1422 current_pstate_driver = &amd_pstate_driver; 1423 1424 return 0; 1425 } 1426 1427 return -EINVAL; 1428 } 1429 early_param("amd_pstate", amd_pstate_param); 1430 1431 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 1432 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 1433