1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/kernel.h> 26 #include <linux/module.h> 27 #include <linux/init.h> 28 #include <linux/smp.h> 29 #include <linux/sched.h> 30 #include <linux/cpufreq.h> 31 #include <linux/compiler.h> 32 #include <linux/dmi.h> 33 #include <linux/slab.h> 34 #include <linux/acpi.h> 35 #include <linux/io.h> 36 #include <linux/delay.h> 37 #include <linux/uaccess.h> 38 #include <linux/static_call.h> 39 #include <linux/amd-pstate.h> 40 41 #include <acpi/processor.h> 42 #include <acpi/cppc_acpi.h> 43 44 #include <asm/msr.h> 45 #include <asm/processor.h> 46 #include <asm/cpufeature.h> 47 #include <asm/cpu_device_id.h> 48 #include "amd-pstate-trace.h" 49 50 #define AMD_PSTATE_TRANSITION_LATENCY 20000 51 #define AMD_PSTATE_TRANSITION_DELAY 1000 52 53 /* 54 * TODO: We need more time to fine tune processors with shared memory solution 55 * with community together. 56 * 57 * There are some performance drops on the CPU benchmarks which reports from 58 * Suse. We are co-working with them to fine tune the shared memory solution. So 59 * we disable it by default to go acpi-cpufreq on these processors and add a 60 * module parameter to be able to enable it manually for debugging. 61 */ 62 static struct cpufreq_driver *current_pstate_driver; 63 static struct cpufreq_driver amd_pstate_driver; 64 static struct cpufreq_driver amd_pstate_epp_driver; 65 static int cppc_state = AMD_PSTATE_UNDEFINED; 66 static bool cppc_enabled; 67 68 /* 69 * AMD Energy Preference Performance (EPP) 70 * The EPP is used in the CCLK DPM controller to drive 71 * the frequency that a core is going to operate during 72 * short periods of activity. EPP values will be utilized for 73 * different OS profiles (balanced, performance, power savings) 74 * display strings corresponding to EPP index in the 75 * energy_perf_strings[] 76 * index String 77 *------------------------------------- 78 * 0 default 79 * 1 performance 80 * 2 balance_performance 81 * 3 balance_power 82 * 4 power 83 */ 84 enum energy_perf_value_index { 85 EPP_INDEX_DEFAULT = 0, 86 EPP_INDEX_PERFORMANCE, 87 EPP_INDEX_BALANCE_PERFORMANCE, 88 EPP_INDEX_BALANCE_POWERSAVE, 89 EPP_INDEX_POWERSAVE, 90 }; 91 92 static const char * const energy_perf_strings[] = { 93 [EPP_INDEX_DEFAULT] = "default", 94 [EPP_INDEX_PERFORMANCE] = "performance", 95 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 96 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 97 [EPP_INDEX_POWERSAVE] = "power", 98 NULL 99 }; 100 101 static unsigned int epp_values[] = { 102 [EPP_INDEX_DEFAULT] = 0, 103 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 104 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 105 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 106 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 107 }; 108 109 typedef int (*cppc_mode_transition_fn)(int); 110 111 static inline int get_mode_idx_from_str(const char *str, size_t size) 112 { 113 int i; 114 115 for (i=0; i < AMD_PSTATE_MAX; i++) { 116 if (!strncmp(str, amd_pstate_mode_string[i], size)) 117 return i; 118 } 119 return -EINVAL; 120 } 121 122 static DEFINE_MUTEX(amd_pstate_limits_lock); 123 static DEFINE_MUTEX(amd_pstate_driver_lock); 124 125 static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) 126 { 127 u64 epp; 128 int ret; 129 130 if (boot_cpu_has(X86_FEATURE_CPPC)) { 131 if (!cppc_req_cached) { 132 epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 133 &cppc_req_cached); 134 if (epp) 135 return epp; 136 } 137 epp = (cppc_req_cached >> 24) & 0xFF; 138 } else { 139 ret = cppc_get_epp_perf(cpudata->cpu, &epp); 140 if (ret < 0) { 141 pr_debug("Could not retrieve energy perf value (%d)\n", ret); 142 return -EIO; 143 } 144 } 145 146 return (s16)(epp & 0xff); 147 } 148 149 static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) 150 { 151 s16 epp; 152 int index = -EINVAL; 153 154 epp = amd_pstate_get_epp(cpudata, 0); 155 if (epp < 0) 156 return epp; 157 158 switch (epp) { 159 case AMD_CPPC_EPP_PERFORMANCE: 160 index = EPP_INDEX_PERFORMANCE; 161 break; 162 case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 163 index = EPP_INDEX_BALANCE_PERFORMANCE; 164 break; 165 case AMD_CPPC_EPP_BALANCE_POWERSAVE: 166 index = EPP_INDEX_BALANCE_POWERSAVE; 167 break; 168 case AMD_CPPC_EPP_POWERSAVE: 169 index = EPP_INDEX_POWERSAVE; 170 break; 171 default: 172 break; 173 } 174 175 return index; 176 } 177 178 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, 179 u32 des_perf, u32 max_perf, bool fast_switch) 180 { 181 if (fast_switch) 182 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); 183 else 184 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 185 READ_ONCE(cpudata->cppc_req_cached)); 186 } 187 188 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); 189 190 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, 191 u32 min_perf, u32 des_perf, 192 u32 max_perf, bool fast_switch) 193 { 194 static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, 195 max_perf, fast_switch); 196 } 197 198 static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) 199 { 200 int ret; 201 struct cppc_perf_ctrls perf_ctrls; 202 203 if (boot_cpu_has(X86_FEATURE_CPPC)) { 204 u64 value = READ_ONCE(cpudata->cppc_req_cached); 205 206 value &= ~GENMASK_ULL(31, 24); 207 value |= (u64)epp << 24; 208 WRITE_ONCE(cpudata->cppc_req_cached, value); 209 210 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 211 if (!ret) 212 cpudata->epp_cached = epp; 213 } else { 214 amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U, 215 cpudata->max_limit_perf, false); 216 217 perf_ctrls.energy_perf = epp; 218 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 219 if (ret) { 220 pr_debug("failed to set energy perf value (%d)\n", ret); 221 return ret; 222 } 223 cpudata->epp_cached = epp; 224 } 225 226 return ret; 227 } 228 229 static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, 230 int pref_index) 231 { 232 int epp = -EINVAL; 233 int ret; 234 235 if (!pref_index) { 236 pr_debug("EPP pref_index is invalid\n"); 237 return -EINVAL; 238 } 239 240 if (epp == -EINVAL) 241 epp = epp_values[pref_index]; 242 243 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 244 pr_debug("EPP cannot be set under performance policy\n"); 245 return -EBUSY; 246 } 247 248 ret = amd_pstate_set_epp(cpudata, epp); 249 250 return ret; 251 } 252 253 static inline int pstate_enable(bool enable) 254 { 255 int ret, cpu; 256 unsigned long logical_proc_id_mask = 0; 257 258 if (enable == cppc_enabled) 259 return 0; 260 261 for_each_present_cpu(cpu) { 262 unsigned long logical_id = topology_logical_die_id(cpu); 263 264 if (test_bit(logical_id, &logical_proc_id_mask)) 265 continue; 266 267 set_bit(logical_id, &logical_proc_id_mask); 268 269 ret = wrmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_ENABLE, 270 enable); 271 if (ret) 272 return ret; 273 } 274 275 cppc_enabled = enable; 276 return 0; 277 } 278 279 static int cppc_enable(bool enable) 280 { 281 int cpu, ret = 0; 282 struct cppc_perf_ctrls perf_ctrls; 283 284 if (enable == cppc_enabled) 285 return 0; 286 287 for_each_present_cpu(cpu) { 288 ret = cppc_set_enable(cpu, enable); 289 if (ret) 290 return ret; 291 292 /* Enable autonomous mode for EPP */ 293 if (cppc_state == AMD_PSTATE_ACTIVE) { 294 /* Set desired perf as zero to allow EPP firmware control */ 295 perf_ctrls.desired_perf = 0; 296 ret = cppc_set_perf(cpu, &perf_ctrls); 297 if (ret) 298 return ret; 299 } 300 } 301 302 cppc_enabled = enable; 303 return ret; 304 } 305 306 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); 307 308 static inline int amd_pstate_enable(bool enable) 309 { 310 return static_call(amd_pstate_enable)(enable); 311 } 312 313 static int pstate_init_perf(struct amd_cpudata *cpudata) 314 { 315 u64 cap1; 316 u32 highest_perf; 317 318 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 319 &cap1); 320 if (ret) 321 return ret; 322 323 /* 324 * TODO: Introduce AMD specific power feature. 325 * 326 * CPPC entry doesn't indicate the highest performance in some ASICs. 327 */ 328 highest_perf = amd_get_highest_perf(); 329 if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) 330 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); 331 332 WRITE_ONCE(cpudata->highest_perf, highest_perf); 333 WRITE_ONCE(cpudata->max_limit_perf, highest_perf); 334 WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); 335 WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); 336 WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); 337 WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1)); 338 return 0; 339 } 340 341 static int cppc_init_perf(struct amd_cpudata *cpudata) 342 { 343 struct cppc_perf_caps cppc_perf; 344 u32 highest_perf; 345 346 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 347 if (ret) 348 return ret; 349 350 highest_perf = amd_get_highest_perf(); 351 if (highest_perf > cppc_perf.highest_perf) 352 highest_perf = cppc_perf.highest_perf; 353 354 WRITE_ONCE(cpudata->highest_perf, highest_perf); 355 WRITE_ONCE(cpudata->max_limit_perf, highest_perf); 356 WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); 357 WRITE_ONCE(cpudata->lowest_nonlinear_perf, 358 cppc_perf.lowest_nonlinear_perf); 359 WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); 360 WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf); 361 362 if (cppc_state == AMD_PSTATE_ACTIVE) 363 return 0; 364 365 ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf); 366 if (ret) { 367 pr_warn("failed to get auto_sel, ret: %d\n", ret); 368 return 0; 369 } 370 371 ret = cppc_set_auto_sel(cpudata->cpu, 372 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 373 374 if (ret) 375 pr_warn("failed to set auto_sel, ret: %d\n", ret); 376 377 return ret; 378 } 379 380 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); 381 382 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 383 { 384 return static_call(amd_pstate_init_perf)(cpudata); 385 } 386 387 static void cppc_update_perf(struct amd_cpudata *cpudata, 388 u32 min_perf, u32 des_perf, 389 u32 max_perf, bool fast_switch) 390 { 391 struct cppc_perf_ctrls perf_ctrls; 392 393 perf_ctrls.max_perf = max_perf; 394 perf_ctrls.min_perf = min_perf; 395 perf_ctrls.desired_perf = des_perf; 396 397 cppc_set_perf(cpudata->cpu, &perf_ctrls); 398 } 399 400 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 401 { 402 u64 aperf, mperf, tsc; 403 unsigned long flags; 404 405 local_irq_save(flags); 406 rdmsrl(MSR_IA32_APERF, aperf); 407 rdmsrl(MSR_IA32_MPERF, mperf); 408 tsc = rdtsc(); 409 410 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 411 local_irq_restore(flags); 412 return false; 413 } 414 415 local_irq_restore(flags); 416 417 cpudata->cur.aperf = aperf; 418 cpudata->cur.mperf = mperf; 419 cpudata->cur.tsc = tsc; 420 cpudata->cur.aperf -= cpudata->prev.aperf; 421 cpudata->cur.mperf -= cpudata->prev.mperf; 422 cpudata->cur.tsc -= cpudata->prev.tsc; 423 424 cpudata->prev.aperf = aperf; 425 cpudata->prev.mperf = mperf; 426 cpudata->prev.tsc = tsc; 427 428 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 429 430 return true; 431 } 432 433 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, 434 u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags) 435 { 436 u64 prev = READ_ONCE(cpudata->cppc_req_cached); 437 u64 value = prev; 438 439 min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, 440 cpudata->max_limit_perf); 441 max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, 442 cpudata->max_limit_perf); 443 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 444 445 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { 446 min_perf = des_perf; 447 des_perf = 0; 448 } 449 450 value &= ~AMD_CPPC_MIN_PERF(~0L); 451 value |= AMD_CPPC_MIN_PERF(min_perf); 452 453 value &= ~AMD_CPPC_DES_PERF(~0L); 454 value |= AMD_CPPC_DES_PERF(des_perf); 455 456 value &= ~AMD_CPPC_MAX_PERF(~0L); 457 value |= AMD_CPPC_MAX_PERF(max_perf); 458 459 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 460 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 461 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 462 cpudata->cpu, (value != prev), fast_switch); 463 } 464 465 if (value == prev) 466 return; 467 468 WRITE_ONCE(cpudata->cppc_req_cached, value); 469 470 amd_pstate_update_perf(cpudata, min_perf, des_perf, 471 max_perf, fast_switch); 472 } 473 474 static int amd_pstate_verify(struct cpufreq_policy_data *policy) 475 { 476 cpufreq_verify_within_cpu_limits(policy); 477 478 return 0; 479 } 480 481 static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) 482 { 483 u32 max_limit_perf, min_limit_perf; 484 struct amd_cpudata *cpudata = policy->driver_data; 485 486 max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); 487 min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); 488 489 WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); 490 WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); 491 WRITE_ONCE(cpudata->max_limit_freq, policy->max); 492 WRITE_ONCE(cpudata->min_limit_freq, policy->min); 493 494 return 0; 495 } 496 497 static int amd_pstate_update_freq(struct cpufreq_policy *policy, 498 unsigned int target_freq, bool fast_switch) 499 { 500 struct cpufreq_freqs freqs; 501 struct amd_cpudata *cpudata = policy->driver_data; 502 unsigned long max_perf, min_perf, des_perf, cap_perf; 503 504 if (!cpudata->max_freq) 505 return -ENODEV; 506 507 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 508 amd_pstate_update_min_max_limit(policy); 509 510 cap_perf = READ_ONCE(cpudata->highest_perf); 511 min_perf = READ_ONCE(cpudata->lowest_perf); 512 max_perf = cap_perf; 513 514 freqs.old = policy->cur; 515 freqs.new = target_freq; 516 517 des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, 518 cpudata->max_freq); 519 520 WARN_ON(fast_switch && !policy->fast_switch_enabled); 521 /* 522 * If fast_switch is desired, then there aren't any registered 523 * transition notifiers. See comment for 524 * cpufreq_enable_fast_switch(). 525 */ 526 if (!fast_switch) 527 cpufreq_freq_transition_begin(policy, &freqs); 528 529 amd_pstate_update(cpudata, min_perf, des_perf, 530 max_perf, fast_switch, policy->governor->flags); 531 532 if (!fast_switch) 533 cpufreq_freq_transition_end(policy, &freqs, false); 534 535 return 0; 536 } 537 538 static int amd_pstate_target(struct cpufreq_policy *policy, 539 unsigned int target_freq, 540 unsigned int relation) 541 { 542 return amd_pstate_update_freq(policy, target_freq, false); 543 } 544 545 static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, 546 unsigned int target_freq) 547 { 548 if (!amd_pstate_update_freq(policy, target_freq, true)) 549 return target_freq; 550 return policy->cur; 551 } 552 553 static void amd_pstate_adjust_perf(unsigned int cpu, 554 unsigned long _min_perf, 555 unsigned long target_perf, 556 unsigned long capacity) 557 { 558 unsigned long max_perf, min_perf, des_perf, 559 cap_perf, lowest_nonlinear_perf, max_freq; 560 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 561 struct amd_cpudata *cpudata = policy->driver_data; 562 unsigned int target_freq; 563 564 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) 565 amd_pstate_update_min_max_limit(policy); 566 567 568 cap_perf = READ_ONCE(cpudata->highest_perf); 569 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); 570 max_freq = READ_ONCE(cpudata->max_freq); 571 572 des_perf = cap_perf; 573 if (target_perf < capacity) 574 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 575 576 min_perf = READ_ONCE(cpudata->lowest_perf); 577 if (_min_perf < capacity) 578 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 579 580 if (min_perf < lowest_nonlinear_perf) 581 min_perf = lowest_nonlinear_perf; 582 583 max_perf = cap_perf; 584 if (max_perf < min_perf) 585 max_perf = min_perf; 586 587 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 588 target_freq = div_u64(des_perf * max_freq, max_perf); 589 policy->cur = target_freq; 590 591 amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, 592 policy->governor->flags); 593 cpufreq_cpu_put(policy); 594 } 595 596 static int amd_get_min_freq(struct amd_cpudata *cpudata) 597 { 598 struct cppc_perf_caps cppc_perf; 599 600 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 601 if (ret) 602 return ret; 603 604 /* Switch to khz */ 605 return cppc_perf.lowest_freq * 1000; 606 } 607 608 static int amd_get_max_freq(struct amd_cpudata *cpudata) 609 { 610 struct cppc_perf_caps cppc_perf; 611 u32 max_perf, max_freq, nominal_freq, nominal_perf; 612 u64 boost_ratio; 613 614 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 615 if (ret) 616 return ret; 617 618 nominal_freq = cppc_perf.nominal_freq; 619 nominal_perf = READ_ONCE(cpudata->nominal_perf); 620 max_perf = READ_ONCE(cpudata->highest_perf); 621 622 boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT, 623 nominal_perf); 624 625 max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT; 626 627 /* Switch to khz */ 628 return max_freq * 1000; 629 } 630 631 static int amd_get_nominal_freq(struct amd_cpudata *cpudata) 632 { 633 struct cppc_perf_caps cppc_perf; 634 635 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 636 if (ret) 637 return ret; 638 639 /* Switch to khz */ 640 return cppc_perf.nominal_freq * 1000; 641 } 642 643 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) 644 { 645 struct cppc_perf_caps cppc_perf; 646 u32 lowest_nonlinear_freq, lowest_nonlinear_perf, 647 nominal_freq, nominal_perf; 648 u64 lowest_nonlinear_ratio; 649 650 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 651 if (ret) 652 return ret; 653 654 nominal_freq = cppc_perf.nominal_freq; 655 nominal_perf = READ_ONCE(cpudata->nominal_perf); 656 657 lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 658 659 lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, 660 nominal_perf); 661 662 lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT; 663 664 /* Switch to khz */ 665 return lowest_nonlinear_freq * 1000; 666 } 667 668 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 669 { 670 struct amd_cpudata *cpudata = policy->driver_data; 671 int ret; 672 673 if (!cpudata->boost_supported) { 674 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 675 return -EINVAL; 676 } 677 678 if (state) 679 policy->cpuinfo.max_freq = cpudata->max_freq; 680 else 681 policy->cpuinfo.max_freq = cpudata->nominal_freq; 682 683 policy->max = policy->cpuinfo.max_freq; 684 685 ret = freq_qos_update_request(&cpudata->req[1], 686 policy->cpuinfo.max_freq); 687 if (ret < 0) 688 return ret; 689 690 return 0; 691 } 692 693 static void amd_pstate_boost_init(struct amd_cpudata *cpudata) 694 { 695 u32 highest_perf, nominal_perf; 696 697 highest_perf = READ_ONCE(cpudata->highest_perf); 698 nominal_perf = READ_ONCE(cpudata->nominal_perf); 699 700 if (highest_perf <= nominal_perf) 701 return; 702 703 cpudata->boost_supported = true; 704 current_pstate_driver->boost_enabled = true; 705 } 706 707 static void amd_perf_ctl_reset(unsigned int cpu) 708 { 709 wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 710 } 711 712 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 713 { 714 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 715 struct device *dev; 716 struct amd_cpudata *cpudata; 717 718 /* 719 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 720 * which is ideal for initialization process. 721 */ 722 amd_perf_ctl_reset(policy->cpu); 723 dev = get_cpu_device(policy->cpu); 724 if (!dev) 725 return -ENODEV; 726 727 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 728 if (!cpudata) 729 return -ENOMEM; 730 731 cpudata->cpu = policy->cpu; 732 733 ret = amd_pstate_init_perf(cpudata); 734 if (ret) 735 goto free_cpudata1; 736 737 min_freq = amd_get_min_freq(cpudata); 738 max_freq = amd_get_max_freq(cpudata); 739 nominal_freq = amd_get_nominal_freq(cpudata); 740 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 741 742 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 743 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 744 min_freq, max_freq); 745 ret = -EINVAL; 746 goto free_cpudata1; 747 } 748 749 policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; 750 policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; 751 752 policy->min = min_freq; 753 policy->max = max_freq; 754 755 policy->cpuinfo.min_freq = min_freq; 756 policy->cpuinfo.max_freq = max_freq; 757 758 /* It will be updated by governor */ 759 policy->cur = policy->cpuinfo.min_freq; 760 761 if (boot_cpu_has(X86_FEATURE_CPPC)) 762 policy->fast_switch_possible = true; 763 764 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 765 FREQ_QOS_MIN, policy->cpuinfo.min_freq); 766 if (ret < 0) { 767 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 768 goto free_cpudata1; 769 } 770 771 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 772 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 773 if (ret < 0) { 774 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 775 goto free_cpudata2; 776 } 777 778 /* Initial processor data capability frequencies */ 779 cpudata->max_freq = max_freq; 780 cpudata->min_freq = min_freq; 781 cpudata->max_limit_freq = max_freq; 782 cpudata->min_limit_freq = min_freq; 783 cpudata->nominal_freq = nominal_freq; 784 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 785 786 policy->driver_data = cpudata; 787 788 amd_pstate_boost_init(cpudata); 789 if (!current_pstate_driver->adjust_perf) 790 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 791 792 return 0; 793 794 free_cpudata2: 795 freq_qos_remove_request(&cpudata->req[0]); 796 free_cpudata1: 797 kfree(cpudata); 798 return ret; 799 } 800 801 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) 802 { 803 struct amd_cpudata *cpudata = policy->driver_data; 804 805 freq_qos_remove_request(&cpudata->req[1]); 806 freq_qos_remove_request(&cpudata->req[0]); 807 policy->fast_switch_possible = false; 808 kfree(cpudata); 809 810 return 0; 811 } 812 813 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) 814 { 815 int ret; 816 817 ret = amd_pstate_enable(true); 818 if (ret) 819 pr_err("failed to enable amd-pstate during resume, return %d\n", ret); 820 821 return ret; 822 } 823 824 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) 825 { 826 int ret; 827 828 ret = amd_pstate_enable(false); 829 if (ret) 830 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); 831 832 return ret; 833 } 834 835 /* Sysfs attributes */ 836 837 /* 838 * This frequency is to indicate the maximum hardware frequency. 839 * If boost is not active but supported, the frequency will be larger than the 840 * one in cpuinfo. 841 */ 842 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 843 char *buf) 844 { 845 int max_freq; 846 struct amd_cpudata *cpudata = policy->driver_data; 847 848 max_freq = amd_get_max_freq(cpudata); 849 if (max_freq < 0) 850 return max_freq; 851 852 return sysfs_emit(buf, "%u\n", max_freq); 853 } 854 855 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 856 char *buf) 857 { 858 int freq; 859 struct amd_cpudata *cpudata = policy->driver_data; 860 861 freq = amd_get_lowest_nonlinear_freq(cpudata); 862 if (freq < 0) 863 return freq; 864 865 return sysfs_emit(buf, "%u\n", freq); 866 } 867 868 /* 869 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 870 * need to expose it to sysfs. 871 */ 872 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 873 char *buf) 874 { 875 u32 perf; 876 struct amd_cpudata *cpudata = policy->driver_data; 877 878 perf = READ_ONCE(cpudata->highest_perf); 879 880 return sysfs_emit(buf, "%u\n", perf); 881 } 882 883 static ssize_t show_energy_performance_available_preferences( 884 struct cpufreq_policy *policy, char *buf) 885 { 886 int i = 0; 887 int offset = 0; 888 struct amd_cpudata *cpudata = policy->driver_data; 889 890 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 891 return sysfs_emit_at(buf, offset, "%s\n", 892 energy_perf_strings[EPP_INDEX_PERFORMANCE]); 893 894 while (energy_perf_strings[i] != NULL) 895 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); 896 897 offset += sysfs_emit_at(buf, offset, "\n"); 898 899 return offset; 900 } 901 902 static ssize_t store_energy_performance_preference( 903 struct cpufreq_policy *policy, const char *buf, size_t count) 904 { 905 struct amd_cpudata *cpudata = policy->driver_data; 906 char str_preference[21]; 907 ssize_t ret; 908 909 ret = sscanf(buf, "%20s", str_preference); 910 if (ret != 1) 911 return -EINVAL; 912 913 ret = match_string(energy_perf_strings, -1, str_preference); 914 if (ret < 0) 915 return -EINVAL; 916 917 mutex_lock(&amd_pstate_limits_lock); 918 ret = amd_pstate_set_energy_pref_index(cpudata, ret); 919 mutex_unlock(&amd_pstate_limits_lock); 920 921 return ret ?: count; 922 } 923 924 static ssize_t show_energy_performance_preference( 925 struct cpufreq_policy *policy, char *buf) 926 { 927 struct amd_cpudata *cpudata = policy->driver_data; 928 int preference; 929 930 preference = amd_pstate_get_energy_pref_index(cpudata); 931 if (preference < 0) 932 return preference; 933 934 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 935 } 936 937 static void amd_pstate_driver_cleanup(void) 938 { 939 amd_pstate_enable(false); 940 cppc_state = AMD_PSTATE_DISABLE; 941 current_pstate_driver = NULL; 942 } 943 944 static int amd_pstate_register_driver(int mode) 945 { 946 int ret; 947 948 if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED) 949 current_pstate_driver = &amd_pstate_driver; 950 else if (mode == AMD_PSTATE_ACTIVE) 951 current_pstate_driver = &amd_pstate_epp_driver; 952 else 953 return -EINVAL; 954 955 cppc_state = mode; 956 ret = cpufreq_register_driver(current_pstate_driver); 957 if (ret) { 958 amd_pstate_driver_cleanup(); 959 return ret; 960 } 961 return 0; 962 } 963 964 static int amd_pstate_unregister_driver(int dummy) 965 { 966 cpufreq_unregister_driver(current_pstate_driver); 967 amd_pstate_driver_cleanup(); 968 return 0; 969 } 970 971 static int amd_pstate_change_mode_without_dvr_change(int mode) 972 { 973 int cpu = 0; 974 975 cppc_state = mode; 976 977 if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) 978 return 0; 979 980 for_each_present_cpu(cpu) { 981 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); 982 } 983 984 return 0; 985 } 986 987 static int amd_pstate_change_driver_mode(int mode) 988 { 989 int ret; 990 991 ret = amd_pstate_unregister_driver(0); 992 if (ret) 993 return ret; 994 995 ret = amd_pstate_register_driver(mode); 996 if (ret) 997 return ret; 998 999 return 0; 1000 } 1001 1002 static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { 1003 [AMD_PSTATE_DISABLE] = { 1004 [AMD_PSTATE_DISABLE] = NULL, 1005 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, 1006 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, 1007 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, 1008 }, 1009 [AMD_PSTATE_PASSIVE] = { 1010 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1011 [AMD_PSTATE_PASSIVE] = NULL, 1012 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1013 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, 1014 }, 1015 [AMD_PSTATE_ACTIVE] = { 1016 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1017 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, 1018 [AMD_PSTATE_ACTIVE] = NULL, 1019 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, 1020 }, 1021 [AMD_PSTATE_GUIDED] = { 1022 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, 1023 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, 1024 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, 1025 [AMD_PSTATE_GUIDED] = NULL, 1026 }, 1027 }; 1028 1029 static ssize_t amd_pstate_show_status(char *buf) 1030 { 1031 if (!current_pstate_driver) 1032 return sysfs_emit(buf, "disable\n"); 1033 1034 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 1035 } 1036 1037 static int amd_pstate_update_status(const char *buf, size_t size) 1038 { 1039 int mode_idx; 1040 1041 if (size > strlen("passive") || size < strlen("active")) 1042 return -EINVAL; 1043 1044 mode_idx = get_mode_idx_from_str(buf, size); 1045 1046 if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX) 1047 return -EINVAL; 1048 1049 if (mode_state_machine[cppc_state][mode_idx]) 1050 return mode_state_machine[cppc_state][mode_idx](mode_idx); 1051 1052 return 0; 1053 } 1054 1055 static ssize_t status_show(struct device *dev, 1056 struct device_attribute *attr, char *buf) 1057 { 1058 ssize_t ret; 1059 1060 mutex_lock(&amd_pstate_driver_lock); 1061 ret = amd_pstate_show_status(buf); 1062 mutex_unlock(&amd_pstate_driver_lock); 1063 1064 return ret; 1065 } 1066 1067 static ssize_t status_store(struct device *a, struct device_attribute *b, 1068 const char *buf, size_t count) 1069 { 1070 char *p = memchr(buf, '\n', count); 1071 int ret; 1072 1073 mutex_lock(&amd_pstate_driver_lock); 1074 ret = amd_pstate_update_status(buf, p ? p - buf : count); 1075 mutex_unlock(&amd_pstate_driver_lock); 1076 1077 return ret < 0 ? ret : count; 1078 } 1079 1080 cpufreq_freq_attr_ro(amd_pstate_max_freq); 1081 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 1082 1083 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 1084 cpufreq_freq_attr_rw(energy_performance_preference); 1085 cpufreq_freq_attr_ro(energy_performance_available_preferences); 1086 static DEVICE_ATTR_RW(status); 1087 1088 static struct freq_attr *amd_pstate_attr[] = { 1089 &amd_pstate_max_freq, 1090 &amd_pstate_lowest_nonlinear_freq, 1091 &amd_pstate_highest_perf, 1092 NULL, 1093 }; 1094 1095 static struct freq_attr *amd_pstate_epp_attr[] = { 1096 &amd_pstate_max_freq, 1097 &amd_pstate_lowest_nonlinear_freq, 1098 &amd_pstate_highest_perf, 1099 &energy_performance_preference, 1100 &energy_performance_available_preferences, 1101 NULL, 1102 }; 1103 1104 static struct attribute *pstate_global_attributes[] = { 1105 &dev_attr_status.attr, 1106 NULL 1107 }; 1108 1109 static const struct attribute_group amd_pstate_global_attr_group = { 1110 .name = "amd_pstate", 1111 .attrs = pstate_global_attributes, 1112 }; 1113 1114 static bool amd_pstate_acpi_pm_profile_server(void) 1115 { 1116 switch (acpi_gbl_FADT.preferred_profile) { 1117 case PM_ENTERPRISE_SERVER: 1118 case PM_SOHO_SERVER: 1119 case PM_PERFORMANCE_SERVER: 1120 return true; 1121 } 1122 return false; 1123 } 1124 1125 static bool amd_pstate_acpi_pm_profile_undefined(void) 1126 { 1127 if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) 1128 return true; 1129 if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) 1130 return true; 1131 return false; 1132 } 1133 1134 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 1135 { 1136 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 1137 struct amd_cpudata *cpudata; 1138 struct device *dev; 1139 u64 value; 1140 1141 /* 1142 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 1143 * which is ideal for initialization process. 1144 */ 1145 amd_perf_ctl_reset(policy->cpu); 1146 dev = get_cpu_device(policy->cpu); 1147 if (!dev) 1148 return -ENODEV; 1149 1150 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 1151 if (!cpudata) 1152 return -ENOMEM; 1153 1154 cpudata->cpu = policy->cpu; 1155 cpudata->epp_policy = 0; 1156 1157 ret = amd_pstate_init_perf(cpudata); 1158 if (ret) 1159 goto free_cpudata1; 1160 1161 min_freq = amd_get_min_freq(cpudata); 1162 max_freq = amd_get_max_freq(cpudata); 1163 nominal_freq = amd_get_nominal_freq(cpudata); 1164 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 1165 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 1166 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 1167 min_freq, max_freq); 1168 ret = -EINVAL; 1169 goto free_cpudata1; 1170 } 1171 1172 policy->cpuinfo.min_freq = min_freq; 1173 policy->cpuinfo.max_freq = max_freq; 1174 /* It will be updated by governor */ 1175 policy->cur = policy->cpuinfo.min_freq; 1176 1177 /* Initial processor data capability frequencies */ 1178 cpudata->max_freq = max_freq; 1179 cpudata->min_freq = min_freq; 1180 cpudata->nominal_freq = nominal_freq; 1181 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 1182 1183 policy->driver_data = cpudata; 1184 1185 cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); 1186 1187 policy->min = policy->cpuinfo.min_freq; 1188 policy->max = policy->cpuinfo.max_freq; 1189 1190 /* 1191 * Set the policy to provide a valid fallback value in case 1192 * the default cpufreq governor is neither powersave nor performance. 1193 */ 1194 if (amd_pstate_acpi_pm_profile_server() || 1195 amd_pstate_acpi_pm_profile_undefined()) 1196 policy->policy = CPUFREQ_POLICY_PERFORMANCE; 1197 else 1198 policy->policy = CPUFREQ_POLICY_POWERSAVE; 1199 1200 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1201 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 1202 if (ret) 1203 return ret; 1204 WRITE_ONCE(cpudata->cppc_req_cached, value); 1205 1206 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); 1207 if (ret) 1208 return ret; 1209 WRITE_ONCE(cpudata->cppc_cap1_cached, value); 1210 } 1211 amd_pstate_boost_init(cpudata); 1212 1213 return 0; 1214 1215 free_cpudata1: 1216 kfree(cpudata); 1217 return ret; 1218 } 1219 1220 static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1221 { 1222 struct amd_cpudata *cpudata = policy->driver_data; 1223 1224 if (cpudata) { 1225 kfree(cpudata); 1226 policy->driver_data = NULL; 1227 } 1228 1229 pr_debug("CPU %d exiting\n", policy->cpu); 1230 return 0; 1231 } 1232 1233 static void amd_pstate_epp_update_limit(struct cpufreq_policy *policy) 1234 { 1235 struct amd_cpudata *cpudata = policy->driver_data; 1236 u32 max_perf, min_perf, min_limit_perf, max_limit_perf; 1237 u64 value; 1238 s16 epp; 1239 1240 max_perf = READ_ONCE(cpudata->highest_perf); 1241 min_perf = READ_ONCE(cpudata->lowest_perf); 1242 max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); 1243 min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); 1244 1245 WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); 1246 WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); 1247 1248 max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, 1249 cpudata->max_limit_perf); 1250 min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, 1251 cpudata->max_limit_perf); 1252 value = READ_ONCE(cpudata->cppc_req_cached); 1253 1254 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1255 min_perf = max_perf; 1256 1257 /* Initial min/max values for CPPC Performance Controls Register */ 1258 value &= ~AMD_CPPC_MIN_PERF(~0L); 1259 value |= AMD_CPPC_MIN_PERF(min_perf); 1260 1261 value &= ~AMD_CPPC_MAX_PERF(~0L); 1262 value |= AMD_CPPC_MAX_PERF(max_perf); 1263 1264 /* CPPC EPP feature require to set zero to the desire perf bit */ 1265 value &= ~AMD_CPPC_DES_PERF(~0L); 1266 value |= AMD_CPPC_DES_PERF(0); 1267 1268 cpudata->epp_policy = cpudata->policy; 1269 1270 /* Get BIOS pre-defined epp value */ 1271 epp = amd_pstate_get_epp(cpudata, value); 1272 if (epp < 0) { 1273 /** 1274 * This return value can only be negative for shared_memory 1275 * systems where EPP register read/write not supported. 1276 */ 1277 return; 1278 } 1279 1280 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1281 epp = 0; 1282 1283 /* Set initial EPP value */ 1284 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1285 value &= ~GENMASK_ULL(31, 24); 1286 value |= (u64)epp << 24; 1287 } 1288 1289 WRITE_ONCE(cpudata->cppc_req_cached, value); 1290 amd_pstate_set_epp(cpudata, epp); 1291 } 1292 1293 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 1294 { 1295 struct amd_cpudata *cpudata = policy->driver_data; 1296 1297 if (!policy->cpuinfo.max_freq) 1298 return -ENODEV; 1299 1300 pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", 1301 policy->cpuinfo.max_freq, policy->max); 1302 1303 cpudata->policy = policy->policy; 1304 1305 amd_pstate_epp_update_limit(policy); 1306 1307 return 0; 1308 } 1309 1310 static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) 1311 { 1312 struct cppc_perf_ctrls perf_ctrls; 1313 u64 value, max_perf; 1314 int ret; 1315 1316 ret = amd_pstate_enable(true); 1317 if (ret) 1318 pr_err("failed to enable amd pstate during resume, return %d\n", ret); 1319 1320 value = READ_ONCE(cpudata->cppc_req_cached); 1321 max_perf = READ_ONCE(cpudata->highest_perf); 1322 1323 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1324 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1325 } else { 1326 perf_ctrls.max_perf = max_perf; 1327 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); 1328 cppc_set_perf(cpudata->cpu, &perf_ctrls); 1329 } 1330 } 1331 1332 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) 1333 { 1334 struct amd_cpudata *cpudata = policy->driver_data; 1335 1336 pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); 1337 1338 if (cppc_state == AMD_PSTATE_ACTIVE) { 1339 amd_pstate_epp_reenable(cpudata); 1340 cpudata->suspended = false; 1341 } 1342 1343 return 0; 1344 } 1345 1346 static void amd_pstate_epp_offline(struct cpufreq_policy *policy) 1347 { 1348 struct amd_cpudata *cpudata = policy->driver_data; 1349 struct cppc_perf_ctrls perf_ctrls; 1350 int min_perf; 1351 u64 value; 1352 1353 min_perf = READ_ONCE(cpudata->lowest_perf); 1354 value = READ_ONCE(cpudata->cppc_req_cached); 1355 1356 mutex_lock(&amd_pstate_limits_lock); 1357 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1358 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; 1359 1360 /* Set max perf same as min perf */ 1361 value &= ~AMD_CPPC_MAX_PERF(~0L); 1362 value |= AMD_CPPC_MAX_PERF(min_perf); 1363 value &= ~AMD_CPPC_MIN_PERF(~0L); 1364 value |= AMD_CPPC_MIN_PERF(min_perf); 1365 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1366 } else { 1367 perf_ctrls.desired_perf = 0; 1368 perf_ctrls.max_perf = min_perf; 1369 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); 1370 cppc_set_perf(cpudata->cpu, &perf_ctrls); 1371 } 1372 mutex_unlock(&amd_pstate_limits_lock); 1373 } 1374 1375 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) 1376 { 1377 struct amd_cpudata *cpudata = policy->driver_data; 1378 1379 pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); 1380 1381 if (cpudata->suspended) 1382 return 0; 1383 1384 if (cppc_state == AMD_PSTATE_ACTIVE) 1385 amd_pstate_epp_offline(policy); 1386 1387 return 0; 1388 } 1389 1390 static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) 1391 { 1392 cpufreq_verify_within_cpu_limits(policy); 1393 pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); 1394 return 0; 1395 } 1396 1397 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) 1398 { 1399 struct amd_cpudata *cpudata = policy->driver_data; 1400 int ret; 1401 1402 /* avoid suspending when EPP is not enabled */ 1403 if (cppc_state != AMD_PSTATE_ACTIVE) 1404 return 0; 1405 1406 /* set this flag to avoid setting core offline*/ 1407 cpudata->suspended = true; 1408 1409 /* disable CPPC in lowlevel firmware */ 1410 ret = amd_pstate_enable(false); 1411 if (ret) 1412 pr_err("failed to suspend, return %d\n", ret); 1413 1414 return 0; 1415 } 1416 1417 static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 1418 { 1419 struct amd_cpudata *cpudata = policy->driver_data; 1420 1421 if (cpudata->suspended) { 1422 mutex_lock(&amd_pstate_limits_lock); 1423 1424 /* enable amd pstate from suspend state*/ 1425 amd_pstate_epp_reenable(cpudata); 1426 1427 mutex_unlock(&amd_pstate_limits_lock); 1428 1429 cpudata->suspended = false; 1430 } 1431 1432 return 0; 1433 } 1434 1435 static struct cpufreq_driver amd_pstate_driver = { 1436 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 1437 .verify = amd_pstate_verify, 1438 .target = amd_pstate_target, 1439 .fast_switch = amd_pstate_fast_switch, 1440 .init = amd_pstate_cpu_init, 1441 .exit = amd_pstate_cpu_exit, 1442 .suspend = amd_pstate_cpu_suspend, 1443 .resume = amd_pstate_cpu_resume, 1444 .set_boost = amd_pstate_set_boost, 1445 .name = "amd-pstate", 1446 .attr = amd_pstate_attr, 1447 }; 1448 1449 static struct cpufreq_driver amd_pstate_epp_driver = { 1450 .flags = CPUFREQ_CONST_LOOPS, 1451 .verify = amd_pstate_epp_verify_policy, 1452 .setpolicy = amd_pstate_epp_set_policy, 1453 .init = amd_pstate_epp_cpu_init, 1454 .exit = amd_pstate_epp_cpu_exit, 1455 .offline = amd_pstate_epp_cpu_offline, 1456 .online = amd_pstate_epp_cpu_online, 1457 .suspend = amd_pstate_epp_suspend, 1458 .resume = amd_pstate_epp_resume, 1459 .name = "amd-pstate-epp", 1460 .attr = amd_pstate_epp_attr, 1461 }; 1462 1463 static int __init amd_pstate_set_driver(int mode_idx) 1464 { 1465 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 1466 cppc_state = mode_idx; 1467 if (cppc_state == AMD_PSTATE_DISABLE) 1468 pr_info("driver is explicitly disabled\n"); 1469 1470 if (cppc_state == AMD_PSTATE_ACTIVE) 1471 current_pstate_driver = &amd_pstate_epp_driver; 1472 1473 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) 1474 current_pstate_driver = &amd_pstate_driver; 1475 1476 return 0; 1477 } 1478 1479 return -EINVAL; 1480 } 1481 1482 static int __init amd_pstate_init(void) 1483 { 1484 struct device *dev_root; 1485 int ret; 1486 1487 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 1488 return -ENODEV; 1489 1490 if (!acpi_cpc_valid()) { 1491 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 1492 return -ENODEV; 1493 } 1494 1495 /* don't keep reloading if cpufreq_driver exists */ 1496 if (cpufreq_get_current_driver()) 1497 return -EEXIST; 1498 1499 switch (cppc_state) { 1500 case AMD_PSTATE_UNDEFINED: 1501 /* Disable on the following configs by default: 1502 * 1. Undefined platforms 1503 * 2. Server platforms 1504 * 3. Shared memory designs 1505 */ 1506 if (amd_pstate_acpi_pm_profile_undefined() || 1507 amd_pstate_acpi_pm_profile_server() || 1508 !boot_cpu_has(X86_FEATURE_CPPC)) { 1509 pr_info("driver load is disabled, boot with specific mode to enable this\n"); 1510 return -ENODEV; 1511 } 1512 ret = amd_pstate_set_driver(CONFIG_X86_AMD_PSTATE_DEFAULT_MODE); 1513 if (ret) 1514 return ret; 1515 break; 1516 case AMD_PSTATE_DISABLE: 1517 return -ENODEV; 1518 case AMD_PSTATE_PASSIVE: 1519 case AMD_PSTATE_ACTIVE: 1520 case AMD_PSTATE_GUIDED: 1521 break; 1522 default: 1523 return -EINVAL; 1524 } 1525 1526 /* capability check */ 1527 if (boot_cpu_has(X86_FEATURE_CPPC)) { 1528 pr_debug("AMD CPPC MSR based functionality is supported\n"); 1529 if (cppc_state != AMD_PSTATE_ACTIVE) 1530 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1531 } else { 1532 pr_debug("AMD CPPC shared memory based functionality is supported\n"); 1533 static_call_update(amd_pstate_enable, cppc_enable); 1534 static_call_update(amd_pstate_init_perf, cppc_init_perf); 1535 static_call_update(amd_pstate_update_perf, cppc_update_perf); 1536 } 1537 1538 /* enable amd pstate feature */ 1539 ret = amd_pstate_enable(true); 1540 if (ret) { 1541 pr_err("failed to enable with return %d\n", ret); 1542 return ret; 1543 } 1544 1545 ret = cpufreq_register_driver(current_pstate_driver); 1546 if (ret) 1547 pr_err("failed to register with return %d\n", ret); 1548 1549 dev_root = bus_get_dev_root(&cpu_subsys); 1550 if (dev_root) { 1551 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); 1552 put_device(dev_root); 1553 if (ret) { 1554 pr_err("sysfs attribute export failed with error %d.\n", ret); 1555 goto global_attr_free; 1556 } 1557 } 1558 1559 return ret; 1560 1561 global_attr_free: 1562 cpufreq_unregister_driver(current_pstate_driver); 1563 return ret; 1564 } 1565 device_initcall(amd_pstate_init); 1566 1567 static int __init amd_pstate_param(char *str) 1568 { 1569 size_t size; 1570 int mode_idx; 1571 1572 if (!str) 1573 return -EINVAL; 1574 1575 size = strlen(str); 1576 mode_idx = get_mode_idx_from_str(str, size); 1577 1578 return amd_pstate_set_driver(mode_idx); 1579 } 1580 early_param("amd_pstate", amd_pstate_param); 1581 1582 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 1583 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 1584