1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * amd-pstate.c - AMD Processor P-state Frequency Driver 4 * 5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6 * 7 * Author: Huang Rui <ray.huang@amd.com> 8 * 9 * AMD P-State introduces a new CPU performance scaling design for AMD 10 * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11 * feature which works with the AMD SMU firmware providing a finer grained 12 * frequency control range. It is to replace the legacy ACPI P-States control, 13 * allows a flexible, low-latency interface for the Linux kernel to directly 14 * communicate the performance hints to hardware. 15 * 16 * AMD P-State is supported on recent AMD Zen base CPU series include some of 17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18 * P-State supported system. And there are two types of hardware implementations 19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21 */ 22 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24 25 #include <linux/kernel.h> 26 #include <linux/module.h> 27 #include <linux/init.h> 28 #include <linux/smp.h> 29 #include <linux/sched.h> 30 #include <linux/cpufreq.h> 31 #include <linux/compiler.h> 32 #include <linux/dmi.h> 33 #include <linux/slab.h> 34 #include <linux/acpi.h> 35 #include <linux/io.h> 36 #include <linux/delay.h> 37 #include <linux/uaccess.h> 38 #include <linux/static_call.h> 39 #include <linux/amd-pstate.h> 40 41 #include <acpi/processor.h> 42 #include <acpi/cppc_acpi.h> 43 44 #include <asm/msr.h> 45 #include <asm/processor.h> 46 #include <asm/cpufeature.h> 47 #include <asm/cpu_device_id.h> 48 #include "amd-pstate-trace.h" 49 50 #define AMD_PSTATE_TRANSITION_LATENCY 20000 51 #define AMD_PSTATE_TRANSITION_DELAY 1000 52 53 /* 54 * TODO: We need more time to fine tune processors with shared memory solution 55 * with community together. 56 * 57 * There are some performance drops on the CPU benchmarks which reports from 58 * Suse. We are co-working with them to fine tune the shared memory solution. So 59 * we disable it by default to go acpi-cpufreq on these processors and add a 60 * module parameter to be able to enable it manually for debugging. 61 */ 62 static bool shared_mem = false; 63 module_param(shared_mem, bool, 0444); 64 MODULE_PARM_DESC(shared_mem, 65 "enable amd-pstate on processors with shared memory solution (false = disabled (default), true = enabled)"); 66 67 static struct cpufreq_driver amd_pstate_driver; 68 69 static inline int pstate_enable(bool enable) 70 { 71 return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); 72 } 73 74 static int cppc_enable(bool enable) 75 { 76 int cpu, ret = 0; 77 78 for_each_present_cpu(cpu) { 79 ret = cppc_set_enable(cpu, enable); 80 if (ret) 81 return ret; 82 } 83 84 return ret; 85 } 86 87 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); 88 89 static inline int amd_pstate_enable(bool enable) 90 { 91 return static_call(amd_pstate_enable)(enable); 92 } 93 94 static int pstate_init_perf(struct amd_cpudata *cpudata) 95 { 96 u64 cap1; 97 u32 highest_perf; 98 99 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 100 &cap1); 101 if (ret) 102 return ret; 103 104 /* 105 * TODO: Introduce AMD specific power feature. 106 * 107 * CPPC entry doesn't indicate the highest performance in some ASICs. 108 */ 109 highest_perf = amd_get_highest_perf(); 110 if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) 111 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); 112 113 WRITE_ONCE(cpudata->highest_perf, highest_perf); 114 115 WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); 116 WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); 117 WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); 118 119 return 0; 120 } 121 122 static int cppc_init_perf(struct amd_cpudata *cpudata) 123 { 124 struct cppc_perf_caps cppc_perf; 125 u32 highest_perf; 126 127 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 128 if (ret) 129 return ret; 130 131 highest_perf = amd_get_highest_perf(); 132 if (highest_perf > cppc_perf.highest_perf) 133 highest_perf = cppc_perf.highest_perf; 134 135 WRITE_ONCE(cpudata->highest_perf, highest_perf); 136 137 WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); 138 WRITE_ONCE(cpudata->lowest_nonlinear_perf, 139 cppc_perf.lowest_nonlinear_perf); 140 WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); 141 142 return 0; 143 } 144 145 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); 146 147 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 148 { 149 return static_call(amd_pstate_init_perf)(cpudata); 150 } 151 152 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, 153 u32 des_perf, u32 max_perf, bool fast_switch) 154 { 155 if (fast_switch) 156 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); 157 else 158 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 159 READ_ONCE(cpudata->cppc_req_cached)); 160 } 161 162 static void cppc_update_perf(struct amd_cpudata *cpudata, 163 u32 min_perf, u32 des_perf, 164 u32 max_perf, bool fast_switch) 165 { 166 struct cppc_perf_ctrls perf_ctrls; 167 168 perf_ctrls.max_perf = max_perf; 169 perf_ctrls.min_perf = min_perf; 170 perf_ctrls.desired_perf = des_perf; 171 172 cppc_set_perf(cpudata->cpu, &perf_ctrls); 173 } 174 175 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); 176 177 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, 178 u32 min_perf, u32 des_perf, 179 u32 max_perf, bool fast_switch) 180 { 181 static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, 182 max_perf, fast_switch); 183 } 184 185 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 186 { 187 u64 aperf, mperf, tsc; 188 unsigned long flags; 189 190 local_irq_save(flags); 191 rdmsrl(MSR_IA32_APERF, aperf); 192 rdmsrl(MSR_IA32_MPERF, mperf); 193 tsc = rdtsc(); 194 195 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 196 local_irq_restore(flags); 197 return false; 198 } 199 200 local_irq_restore(flags); 201 202 cpudata->cur.aperf = aperf; 203 cpudata->cur.mperf = mperf; 204 cpudata->cur.tsc = tsc; 205 cpudata->cur.aperf -= cpudata->prev.aperf; 206 cpudata->cur.mperf -= cpudata->prev.mperf; 207 cpudata->cur.tsc -= cpudata->prev.tsc; 208 209 cpudata->prev.aperf = aperf; 210 cpudata->prev.mperf = mperf; 211 cpudata->prev.tsc = tsc; 212 213 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 214 215 return true; 216 } 217 218 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, 219 u32 des_perf, u32 max_perf, bool fast_switch) 220 { 221 u64 prev = READ_ONCE(cpudata->cppc_req_cached); 222 u64 value = prev; 223 224 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 225 value &= ~AMD_CPPC_MIN_PERF(~0L); 226 value |= AMD_CPPC_MIN_PERF(min_perf); 227 228 value &= ~AMD_CPPC_DES_PERF(~0L); 229 value |= AMD_CPPC_DES_PERF(des_perf); 230 231 value &= ~AMD_CPPC_MAX_PERF(~0L); 232 value |= AMD_CPPC_MAX_PERF(max_perf); 233 234 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 235 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 236 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 237 cpudata->cpu, (value != prev), fast_switch); 238 } 239 240 if (value == prev) 241 return; 242 243 WRITE_ONCE(cpudata->cppc_req_cached, value); 244 245 amd_pstate_update_perf(cpudata, min_perf, des_perf, 246 max_perf, fast_switch); 247 } 248 249 static int amd_pstate_verify(struct cpufreq_policy_data *policy) 250 { 251 cpufreq_verify_within_cpu_limits(policy); 252 253 return 0; 254 } 255 256 static int amd_pstate_target(struct cpufreq_policy *policy, 257 unsigned int target_freq, 258 unsigned int relation) 259 { 260 struct cpufreq_freqs freqs; 261 struct amd_cpudata *cpudata = policy->driver_data; 262 unsigned long max_perf, min_perf, des_perf, cap_perf; 263 264 if (!cpudata->max_freq) 265 return -ENODEV; 266 267 cap_perf = READ_ONCE(cpudata->highest_perf); 268 min_perf = READ_ONCE(cpudata->lowest_perf); 269 max_perf = cap_perf; 270 271 freqs.old = policy->cur; 272 freqs.new = target_freq; 273 274 des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, 275 cpudata->max_freq); 276 277 cpufreq_freq_transition_begin(policy, &freqs); 278 amd_pstate_update(cpudata, min_perf, des_perf, 279 max_perf, false); 280 cpufreq_freq_transition_end(policy, &freqs, false); 281 282 return 0; 283 } 284 285 static void amd_pstate_adjust_perf(unsigned int cpu, 286 unsigned long _min_perf, 287 unsigned long target_perf, 288 unsigned long capacity) 289 { 290 unsigned long max_perf, min_perf, des_perf, 291 cap_perf, lowest_nonlinear_perf; 292 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 293 struct amd_cpudata *cpudata = policy->driver_data; 294 295 cap_perf = READ_ONCE(cpudata->highest_perf); 296 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); 297 298 des_perf = cap_perf; 299 if (target_perf < capacity) 300 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 301 302 min_perf = READ_ONCE(cpudata->highest_perf); 303 if (_min_perf < capacity) 304 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 305 306 if (min_perf < lowest_nonlinear_perf) 307 min_perf = lowest_nonlinear_perf; 308 309 max_perf = cap_perf; 310 if (max_perf < min_perf) 311 max_perf = min_perf; 312 313 amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); 314 } 315 316 static int amd_get_min_freq(struct amd_cpudata *cpudata) 317 { 318 struct cppc_perf_caps cppc_perf; 319 320 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 321 if (ret) 322 return ret; 323 324 /* Switch to khz */ 325 return cppc_perf.lowest_freq * 1000; 326 } 327 328 static int amd_get_max_freq(struct amd_cpudata *cpudata) 329 { 330 struct cppc_perf_caps cppc_perf; 331 u32 max_perf, max_freq, nominal_freq, nominal_perf; 332 u64 boost_ratio; 333 334 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 335 if (ret) 336 return ret; 337 338 nominal_freq = cppc_perf.nominal_freq; 339 nominal_perf = READ_ONCE(cpudata->nominal_perf); 340 max_perf = READ_ONCE(cpudata->highest_perf); 341 342 boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT, 343 nominal_perf); 344 345 max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT; 346 347 /* Switch to khz */ 348 return max_freq * 1000; 349 } 350 351 static int amd_get_nominal_freq(struct amd_cpudata *cpudata) 352 { 353 struct cppc_perf_caps cppc_perf; 354 355 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 356 if (ret) 357 return ret; 358 359 /* Switch to khz */ 360 return cppc_perf.nominal_freq * 1000; 361 } 362 363 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) 364 { 365 struct cppc_perf_caps cppc_perf; 366 u32 lowest_nonlinear_freq, lowest_nonlinear_perf, 367 nominal_freq, nominal_perf; 368 u64 lowest_nonlinear_ratio; 369 370 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 371 if (ret) 372 return ret; 373 374 nominal_freq = cppc_perf.nominal_freq; 375 nominal_perf = READ_ONCE(cpudata->nominal_perf); 376 377 lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 378 379 lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, 380 nominal_perf); 381 382 lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT; 383 384 /* Switch to khz */ 385 return lowest_nonlinear_freq * 1000; 386 } 387 388 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 389 { 390 struct amd_cpudata *cpudata = policy->driver_data; 391 int ret; 392 393 if (!cpudata->boost_supported) { 394 pr_err("Boost mode is not supported by this processor or SBIOS\n"); 395 return -EINVAL; 396 } 397 398 if (state) 399 policy->cpuinfo.max_freq = cpudata->max_freq; 400 else 401 policy->cpuinfo.max_freq = cpudata->nominal_freq; 402 403 policy->max = policy->cpuinfo.max_freq; 404 405 ret = freq_qos_update_request(&cpudata->req[1], 406 policy->cpuinfo.max_freq); 407 if (ret < 0) 408 return ret; 409 410 return 0; 411 } 412 413 static void amd_pstate_boost_init(struct amd_cpudata *cpudata) 414 { 415 u32 highest_perf, nominal_perf; 416 417 highest_perf = READ_ONCE(cpudata->highest_perf); 418 nominal_perf = READ_ONCE(cpudata->nominal_perf); 419 420 if (highest_perf <= nominal_perf) 421 return; 422 423 cpudata->boost_supported = true; 424 amd_pstate_driver.boost_enabled = true; 425 } 426 427 static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 428 { 429 int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 430 struct device *dev; 431 struct amd_cpudata *cpudata; 432 433 dev = get_cpu_device(policy->cpu); 434 if (!dev) 435 return -ENODEV; 436 437 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 438 if (!cpudata) 439 return -ENOMEM; 440 441 cpudata->cpu = policy->cpu; 442 443 ret = amd_pstate_init_perf(cpudata); 444 if (ret) 445 goto free_cpudata1; 446 447 min_freq = amd_get_min_freq(cpudata); 448 max_freq = amd_get_max_freq(cpudata); 449 nominal_freq = amd_get_nominal_freq(cpudata); 450 lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 451 452 if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 453 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 454 min_freq, max_freq); 455 ret = -EINVAL; 456 goto free_cpudata1; 457 } 458 459 policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; 460 policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; 461 462 policy->min = min_freq; 463 policy->max = max_freq; 464 465 policy->cpuinfo.min_freq = min_freq; 466 policy->cpuinfo.max_freq = max_freq; 467 468 /* It will be updated by governor */ 469 policy->cur = policy->cpuinfo.min_freq; 470 471 if (boot_cpu_has(X86_FEATURE_CPPC)) 472 policy->fast_switch_possible = true; 473 474 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 475 FREQ_QOS_MIN, policy->cpuinfo.min_freq); 476 if (ret < 0) { 477 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 478 goto free_cpudata1; 479 } 480 481 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 482 FREQ_QOS_MAX, policy->cpuinfo.max_freq); 483 if (ret < 0) { 484 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 485 goto free_cpudata2; 486 } 487 488 /* Initial processor data capability frequencies */ 489 cpudata->max_freq = max_freq; 490 cpudata->min_freq = min_freq; 491 cpudata->nominal_freq = nominal_freq; 492 cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 493 494 policy->driver_data = cpudata; 495 496 amd_pstate_boost_init(cpudata); 497 498 return 0; 499 500 free_cpudata2: 501 freq_qos_remove_request(&cpudata->req[0]); 502 free_cpudata1: 503 kfree(cpudata); 504 return ret; 505 } 506 507 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) 508 { 509 struct amd_cpudata *cpudata = policy->driver_data; 510 511 freq_qos_remove_request(&cpudata->req[1]); 512 freq_qos_remove_request(&cpudata->req[0]); 513 kfree(cpudata); 514 515 return 0; 516 } 517 518 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) 519 { 520 int ret; 521 522 ret = amd_pstate_enable(true); 523 if (ret) 524 pr_err("failed to enable amd-pstate during resume, return %d\n", ret); 525 526 return ret; 527 } 528 529 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) 530 { 531 int ret; 532 533 ret = amd_pstate_enable(false); 534 if (ret) 535 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); 536 537 return ret; 538 } 539 540 /* Sysfs attributes */ 541 542 /* 543 * This frequency is to indicate the maximum hardware frequency. 544 * If boost is not active but supported, the frequency will be larger than the 545 * one in cpuinfo. 546 */ 547 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 548 char *buf) 549 { 550 int max_freq; 551 struct amd_cpudata *cpudata = policy->driver_data; 552 553 max_freq = amd_get_max_freq(cpudata); 554 if (max_freq < 0) 555 return max_freq; 556 557 return sprintf(&buf[0], "%u\n", max_freq); 558 } 559 560 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 561 char *buf) 562 { 563 int freq; 564 struct amd_cpudata *cpudata = policy->driver_data; 565 566 freq = amd_get_lowest_nonlinear_freq(cpudata); 567 if (freq < 0) 568 return freq; 569 570 return sprintf(&buf[0], "%u\n", freq); 571 } 572 573 /* 574 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 575 * need to expose it to sysfs. 576 */ 577 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 578 char *buf) 579 { 580 u32 perf; 581 struct amd_cpudata *cpudata = policy->driver_data; 582 583 perf = READ_ONCE(cpudata->highest_perf); 584 585 return sprintf(&buf[0], "%u\n", perf); 586 } 587 588 cpufreq_freq_attr_ro(amd_pstate_max_freq); 589 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 590 591 cpufreq_freq_attr_ro(amd_pstate_highest_perf); 592 593 static struct freq_attr *amd_pstate_attr[] = { 594 &amd_pstate_max_freq, 595 &amd_pstate_lowest_nonlinear_freq, 596 &amd_pstate_highest_perf, 597 NULL, 598 }; 599 600 static struct cpufreq_driver amd_pstate_driver = { 601 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 602 .verify = amd_pstate_verify, 603 .target = amd_pstate_target, 604 .init = amd_pstate_cpu_init, 605 .exit = amd_pstate_cpu_exit, 606 .suspend = amd_pstate_cpu_suspend, 607 .resume = amd_pstate_cpu_resume, 608 .set_boost = amd_pstate_set_boost, 609 .name = "amd-pstate", 610 .attr = amd_pstate_attr, 611 }; 612 613 static int __init amd_pstate_init(void) 614 { 615 int ret; 616 617 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 618 return -ENODEV; 619 620 if (!acpi_cpc_valid()) { 621 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 622 return -ENODEV; 623 } 624 625 /* don't keep reloading if cpufreq_driver exists */ 626 if (cpufreq_get_current_driver()) 627 return -EEXIST; 628 629 /* capability check */ 630 if (boot_cpu_has(X86_FEATURE_CPPC)) { 631 pr_debug("AMD CPPC MSR based functionality is supported\n"); 632 amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf; 633 } else if (shared_mem) { 634 static_call_update(amd_pstate_enable, cppc_enable); 635 static_call_update(amd_pstate_init_perf, cppc_init_perf); 636 static_call_update(amd_pstate_update_perf, cppc_update_perf); 637 } else { 638 pr_info("This processor supports shared memory solution, you can enable it with amd_pstate.shared_mem=1\n"); 639 return -ENODEV; 640 } 641 642 /* enable amd pstate feature */ 643 ret = amd_pstate_enable(true); 644 if (ret) { 645 pr_err("failed to enable amd-pstate with return %d\n", ret); 646 return ret; 647 } 648 649 ret = cpufreq_register_driver(&amd_pstate_driver); 650 if (ret) 651 pr_err("failed to register amd_pstate_driver with return %d\n", 652 ret); 653 654 return ret; 655 } 656 657 static void __exit amd_pstate_exit(void) 658 { 659 cpufreq_unregister_driver(&amd_pstate_driver); 660 661 amd_pstate_enable(false); 662 } 663 664 module_init(amd_pstate_init); 665 module_exit(amd_pstate_exit); 666 667 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 668 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 669 MODULE_LICENSE("GPL"); 670