1ec437d71SHuang Rui // SPDX-License-Identifier: GPL-2.0-or-later 2ec437d71SHuang Rui /* 3ec437d71SHuang Rui * amd-pstate.c - AMD Processor P-state Frequency Driver 4ec437d71SHuang Rui * 5ec437d71SHuang Rui * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6ec437d71SHuang Rui * 7ec437d71SHuang Rui * Author: Huang Rui <ray.huang@amd.com> 8ec437d71SHuang Rui * 9ec437d71SHuang Rui * AMD P-State introduces a new CPU performance scaling design for AMD 10ec437d71SHuang Rui * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11ec437d71SHuang Rui * feature which works with the AMD SMU firmware providing a finer grained 12ec437d71SHuang Rui * frequency control range. It is to replace the legacy ACPI P-States control, 13ec437d71SHuang Rui * allows a flexible, low-latency interface for the Linux kernel to directly 14ec437d71SHuang Rui * communicate the performance hints to hardware. 15ec437d71SHuang Rui * 16ec437d71SHuang Rui * AMD P-State is supported on recent AMD Zen base CPU series include some of 17ec437d71SHuang Rui * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18ec437d71SHuang Rui * P-State supported system. And there are two types of hardware implementations 19ec437d71SHuang Rui * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20ec437d71SHuang Rui * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21ec437d71SHuang Rui */ 22ec437d71SHuang Rui 23ec437d71SHuang Rui #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24ec437d71SHuang Rui 25ec437d71SHuang Rui #include <linux/kernel.h> 26ec437d71SHuang Rui #include <linux/module.h> 27ec437d71SHuang Rui #include <linux/init.h> 28ec437d71SHuang Rui #include <linux/smp.h> 29ec437d71SHuang Rui #include <linux/sched.h> 30ec437d71SHuang Rui #include <linux/cpufreq.h> 31ec437d71SHuang Rui #include <linux/compiler.h> 32ec437d71SHuang Rui #include <linux/dmi.h> 33ec437d71SHuang Rui #include <linux/slab.h> 34ec437d71SHuang Rui #include <linux/acpi.h> 35ec437d71SHuang Rui #include <linux/io.h> 36ec437d71SHuang Rui #include <linux/delay.h> 37ec437d71SHuang Rui #include <linux/uaccess.h> 38ec437d71SHuang Rui #include <linux/static_call.h> 39f1375ec1SMeng Li #include <linux/amd-pstate.h> 40ec437d71SHuang Rui 41ec437d71SHuang Rui #include <acpi/processor.h> 42ec437d71SHuang Rui #include <acpi/cppc_acpi.h> 43ec437d71SHuang Rui 44ec437d71SHuang Rui #include <asm/msr.h> 45ec437d71SHuang Rui #include <asm/processor.h> 46ec437d71SHuang Rui #include <asm/cpufeature.h> 47ec437d71SHuang Rui #include <asm/cpu_device_id.h> 4860e10f89SHuang Rui #include "amd-pstate-trace.h" 49ec437d71SHuang Rui 50ca08e46dSPerry Yuan #define AMD_PSTATE_TRANSITION_LATENCY 20000 51ca08e46dSPerry Yuan #define AMD_PSTATE_TRANSITION_DELAY 1000 52ec437d71SHuang Rui 53e059c184SHuang Rui /* 54e059c184SHuang Rui * TODO: We need more time to fine tune processors with shared memory solution 55e059c184SHuang Rui * with community together. 56e059c184SHuang Rui * 57e059c184SHuang Rui * There are some performance drops on the CPU benchmarks which reports from 58e059c184SHuang Rui * Suse. We are co-working with them to fine tune the shared memory solution. So 59e059c184SHuang Rui * we disable it by default to go acpi-cpufreq on these processors and add a 60e059c184SHuang Rui * module parameter to be able to enable it manually for debugging. 61e059c184SHuang Rui */ 62ffa5096aSPerry Yuan static struct cpufreq_driver *current_pstate_driver; 63ec437d71SHuang Rui static struct cpufreq_driver amd_pstate_driver; 64ffa5096aSPerry Yuan static struct cpufreq_driver amd_pstate_epp_driver; 6536c5014eSWyes Karny static int cppc_state = AMD_PSTATE_DISABLE; 66abd61c08SPerry Yuan struct kobject *amd_pstate_kobj; 6736c5014eSWyes Karny 68ffa5096aSPerry Yuan /* 69ffa5096aSPerry Yuan * AMD Energy Preference Performance (EPP) 70ffa5096aSPerry Yuan * The EPP is used in the CCLK DPM controller to drive 71ffa5096aSPerry Yuan * the frequency that a core is going to operate during 72ffa5096aSPerry Yuan * short periods of activity. EPP values will be utilized for 73ffa5096aSPerry Yuan * different OS profiles (balanced, performance, power savings) 74ffa5096aSPerry Yuan * display strings corresponding to EPP index in the 75ffa5096aSPerry Yuan * energy_perf_strings[] 76ffa5096aSPerry Yuan * index String 77ffa5096aSPerry Yuan *------------------------------------- 78ffa5096aSPerry Yuan * 0 default 79ffa5096aSPerry Yuan * 1 performance 80ffa5096aSPerry Yuan * 2 balance_performance 81ffa5096aSPerry Yuan * 3 balance_power 82ffa5096aSPerry Yuan * 4 power 83ffa5096aSPerry Yuan */ 84ffa5096aSPerry Yuan enum energy_perf_value_index { 85ffa5096aSPerry Yuan EPP_INDEX_DEFAULT = 0, 86ffa5096aSPerry Yuan EPP_INDEX_PERFORMANCE, 87ffa5096aSPerry Yuan EPP_INDEX_BALANCE_PERFORMANCE, 88ffa5096aSPerry Yuan EPP_INDEX_BALANCE_POWERSAVE, 89ffa5096aSPerry Yuan EPP_INDEX_POWERSAVE, 90ffa5096aSPerry Yuan }; 91ffa5096aSPerry Yuan 92ffa5096aSPerry Yuan static const char * const energy_perf_strings[] = { 93ffa5096aSPerry Yuan [EPP_INDEX_DEFAULT] = "default", 94ffa5096aSPerry Yuan [EPP_INDEX_PERFORMANCE] = "performance", 95ffa5096aSPerry Yuan [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", 96ffa5096aSPerry Yuan [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", 97ffa5096aSPerry Yuan [EPP_INDEX_POWERSAVE] = "power", 98ffa5096aSPerry Yuan NULL 99ffa5096aSPerry Yuan }; 100ffa5096aSPerry Yuan 101ffa5096aSPerry Yuan static unsigned int epp_values[] = { 102ffa5096aSPerry Yuan [EPP_INDEX_DEFAULT] = 0, 103ffa5096aSPerry Yuan [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, 104ffa5096aSPerry Yuan [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, 105ffa5096aSPerry Yuan [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, 106ffa5096aSPerry Yuan [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, 107ffa5096aSPerry Yuan }; 108ffa5096aSPerry Yuan 10936c5014eSWyes Karny static inline int get_mode_idx_from_str(const char *str, size_t size) 11036c5014eSWyes Karny { 11136c5014eSWyes Karny int i; 11236c5014eSWyes Karny 11336c5014eSWyes Karny for (i=0; i < AMD_PSTATE_MAX; i++) { 11436c5014eSWyes Karny if (!strncmp(str, amd_pstate_mode_string[i], size)) 11536c5014eSWyes Karny return i; 11636c5014eSWyes Karny } 11736c5014eSWyes Karny return -EINVAL; 11836c5014eSWyes Karny } 119ec437d71SHuang Rui 120ffa5096aSPerry Yuan static DEFINE_MUTEX(amd_pstate_limits_lock); 121ffa5096aSPerry Yuan static DEFINE_MUTEX(amd_pstate_driver_lock); 122ffa5096aSPerry Yuan 123ffa5096aSPerry Yuan static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) 124ffa5096aSPerry Yuan { 125ffa5096aSPerry Yuan u64 epp; 126ffa5096aSPerry Yuan int ret; 127ffa5096aSPerry Yuan 128ffa5096aSPerry Yuan if (boot_cpu_has(X86_FEATURE_CPPC)) { 129ffa5096aSPerry Yuan if (!cppc_req_cached) { 130ffa5096aSPerry Yuan epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 131ffa5096aSPerry Yuan &cppc_req_cached); 132ffa5096aSPerry Yuan if (epp) 133ffa5096aSPerry Yuan return epp; 134ffa5096aSPerry Yuan } 135ffa5096aSPerry Yuan epp = (cppc_req_cached >> 24) & 0xFF; 136ffa5096aSPerry Yuan } else { 137ffa5096aSPerry Yuan ret = cppc_get_epp_perf(cpudata->cpu, &epp); 138ffa5096aSPerry Yuan if (ret < 0) { 139ffa5096aSPerry Yuan pr_debug("Could not retrieve energy perf value (%d)\n", ret); 140ffa5096aSPerry Yuan return -EIO; 141ffa5096aSPerry Yuan } 142ffa5096aSPerry Yuan } 143ffa5096aSPerry Yuan 144ffa5096aSPerry Yuan return (s16)(epp & 0xff); 145ffa5096aSPerry Yuan } 146ffa5096aSPerry Yuan 147ffa5096aSPerry Yuan static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) 148ffa5096aSPerry Yuan { 149ffa5096aSPerry Yuan s16 epp; 150ffa5096aSPerry Yuan int index = -EINVAL; 151ffa5096aSPerry Yuan 152ffa5096aSPerry Yuan epp = amd_pstate_get_epp(cpudata, 0); 153ffa5096aSPerry Yuan if (epp < 0) 154ffa5096aSPerry Yuan return epp; 155ffa5096aSPerry Yuan 156ffa5096aSPerry Yuan switch (epp) { 157ffa5096aSPerry Yuan case AMD_CPPC_EPP_PERFORMANCE: 158ffa5096aSPerry Yuan index = EPP_INDEX_PERFORMANCE; 159ffa5096aSPerry Yuan break; 160ffa5096aSPerry Yuan case AMD_CPPC_EPP_BALANCE_PERFORMANCE: 161ffa5096aSPerry Yuan index = EPP_INDEX_BALANCE_PERFORMANCE; 162ffa5096aSPerry Yuan break; 163ffa5096aSPerry Yuan case AMD_CPPC_EPP_BALANCE_POWERSAVE: 164ffa5096aSPerry Yuan index = EPP_INDEX_BALANCE_POWERSAVE; 165ffa5096aSPerry Yuan break; 166ffa5096aSPerry Yuan case AMD_CPPC_EPP_POWERSAVE: 167ffa5096aSPerry Yuan index = EPP_INDEX_POWERSAVE; 168ffa5096aSPerry Yuan break; 169ffa5096aSPerry Yuan default: 170ffa5096aSPerry Yuan break; 171ffa5096aSPerry Yuan } 172ffa5096aSPerry Yuan 173ffa5096aSPerry Yuan return index; 174ffa5096aSPerry Yuan } 175ffa5096aSPerry Yuan 176ffa5096aSPerry Yuan static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) 177ffa5096aSPerry Yuan { 178ffa5096aSPerry Yuan int ret; 179ffa5096aSPerry Yuan struct cppc_perf_ctrls perf_ctrls; 180ffa5096aSPerry Yuan 181ffa5096aSPerry Yuan if (boot_cpu_has(X86_FEATURE_CPPC)) { 182ffa5096aSPerry Yuan u64 value = READ_ONCE(cpudata->cppc_req_cached); 183ffa5096aSPerry Yuan 184ffa5096aSPerry Yuan value &= ~GENMASK_ULL(31, 24); 185ffa5096aSPerry Yuan value |= (u64)epp << 24; 186ffa5096aSPerry Yuan WRITE_ONCE(cpudata->cppc_req_cached, value); 187ffa5096aSPerry Yuan 188ffa5096aSPerry Yuan ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 189ffa5096aSPerry Yuan if (!ret) 190ffa5096aSPerry Yuan cpudata->epp_cached = epp; 191ffa5096aSPerry Yuan } else { 192ffa5096aSPerry Yuan perf_ctrls.energy_perf = epp; 193ffa5096aSPerry Yuan ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); 194ffa5096aSPerry Yuan if (ret) { 195ffa5096aSPerry Yuan pr_debug("failed to set energy perf value (%d)\n", ret); 196ffa5096aSPerry Yuan return ret; 197ffa5096aSPerry Yuan } 198ffa5096aSPerry Yuan cpudata->epp_cached = epp; 199ffa5096aSPerry Yuan } 200ffa5096aSPerry Yuan 201ffa5096aSPerry Yuan return ret; 202ffa5096aSPerry Yuan } 203ffa5096aSPerry Yuan 204ffa5096aSPerry Yuan static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, 205ffa5096aSPerry Yuan int pref_index) 206ffa5096aSPerry Yuan { 207ffa5096aSPerry Yuan int epp = -EINVAL; 208ffa5096aSPerry Yuan int ret; 209ffa5096aSPerry Yuan 210ffa5096aSPerry Yuan if (!pref_index) { 211ffa5096aSPerry Yuan pr_debug("EPP pref_index is invalid\n"); 212ffa5096aSPerry Yuan return -EINVAL; 213ffa5096aSPerry Yuan } 214ffa5096aSPerry Yuan 215ffa5096aSPerry Yuan if (epp == -EINVAL) 216ffa5096aSPerry Yuan epp = epp_values[pref_index]; 217ffa5096aSPerry Yuan 218ffa5096aSPerry Yuan if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 219ffa5096aSPerry Yuan pr_debug("EPP cannot be set under performance policy\n"); 220ffa5096aSPerry Yuan return -EBUSY; 221ffa5096aSPerry Yuan } 222ffa5096aSPerry Yuan 223ffa5096aSPerry Yuan ret = amd_pstate_set_epp(cpudata, epp); 224ffa5096aSPerry Yuan 225ffa5096aSPerry Yuan return ret; 226ffa5096aSPerry Yuan } 227ffa5096aSPerry Yuan 228e059c184SHuang Rui static inline int pstate_enable(bool enable) 229ec437d71SHuang Rui { 230ec437d71SHuang Rui return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); 231ec437d71SHuang Rui } 232ec437d71SHuang Rui 233e059c184SHuang Rui static int cppc_enable(bool enable) 234e059c184SHuang Rui { 235e059c184SHuang Rui int cpu, ret = 0; 236ffa5096aSPerry Yuan struct cppc_perf_ctrls perf_ctrls; 237e059c184SHuang Rui 238e059c184SHuang Rui for_each_present_cpu(cpu) { 239e059c184SHuang Rui ret = cppc_set_enable(cpu, enable); 240e059c184SHuang Rui if (ret) 241e059c184SHuang Rui return ret; 242ffa5096aSPerry Yuan 243ffa5096aSPerry Yuan /* Enable autonomous mode for EPP */ 244ffa5096aSPerry Yuan if (cppc_state == AMD_PSTATE_ACTIVE) { 245ffa5096aSPerry Yuan /* Set desired perf as zero to allow EPP firmware control */ 246ffa5096aSPerry Yuan perf_ctrls.desired_perf = 0; 247ffa5096aSPerry Yuan ret = cppc_set_perf(cpu, &perf_ctrls); 248ffa5096aSPerry Yuan if (ret) 249ffa5096aSPerry Yuan return ret; 250ffa5096aSPerry Yuan } 251e059c184SHuang Rui } 252e059c184SHuang Rui 253e059c184SHuang Rui return ret; 254e059c184SHuang Rui } 255e059c184SHuang Rui 256e059c184SHuang Rui DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); 257e059c184SHuang Rui 258e059c184SHuang Rui static inline int amd_pstate_enable(bool enable) 259e059c184SHuang Rui { 260e059c184SHuang Rui return static_call(amd_pstate_enable)(enable); 261e059c184SHuang Rui } 262e059c184SHuang Rui 263e059c184SHuang Rui static int pstate_init_perf(struct amd_cpudata *cpudata) 264ec437d71SHuang Rui { 265ec437d71SHuang Rui u64 cap1; 266bedadcfbSPerry Yuan u32 highest_perf; 267ec437d71SHuang Rui 268ec437d71SHuang Rui int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 269ec437d71SHuang Rui &cap1); 270ec437d71SHuang Rui if (ret) 271ec437d71SHuang Rui return ret; 272ec437d71SHuang Rui 273ec437d71SHuang Rui /* 274ec437d71SHuang Rui * TODO: Introduce AMD specific power feature. 275ec437d71SHuang Rui * 276ec437d71SHuang Rui * CPPC entry doesn't indicate the highest performance in some ASICs. 277ec437d71SHuang Rui */ 278bedadcfbSPerry Yuan highest_perf = amd_get_highest_perf(); 279bedadcfbSPerry Yuan if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) 280bedadcfbSPerry Yuan highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); 281bedadcfbSPerry Yuan 282bedadcfbSPerry Yuan WRITE_ONCE(cpudata->highest_perf, highest_perf); 283ec437d71SHuang Rui 284ec437d71SHuang Rui WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); 285ec437d71SHuang Rui WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); 286ec437d71SHuang Rui WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); 287ec437d71SHuang Rui 288ec437d71SHuang Rui return 0; 289ec437d71SHuang Rui } 290ec437d71SHuang Rui 291e059c184SHuang Rui static int cppc_init_perf(struct amd_cpudata *cpudata) 292e059c184SHuang Rui { 293e059c184SHuang Rui struct cppc_perf_caps cppc_perf; 294bedadcfbSPerry Yuan u32 highest_perf; 295e059c184SHuang Rui 296e059c184SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 297e059c184SHuang Rui if (ret) 298e059c184SHuang Rui return ret; 299e059c184SHuang Rui 300bedadcfbSPerry Yuan highest_perf = amd_get_highest_perf(); 301bedadcfbSPerry Yuan if (highest_perf > cppc_perf.highest_perf) 302bedadcfbSPerry Yuan highest_perf = cppc_perf.highest_perf; 303bedadcfbSPerry Yuan 304bedadcfbSPerry Yuan WRITE_ONCE(cpudata->highest_perf, highest_perf); 305e059c184SHuang Rui 306e059c184SHuang Rui WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); 307e059c184SHuang Rui WRITE_ONCE(cpudata->lowest_nonlinear_perf, 308e059c184SHuang Rui cppc_perf.lowest_nonlinear_perf); 309e059c184SHuang Rui WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); 310e059c184SHuang Rui 311e059c184SHuang Rui return 0; 312e059c184SHuang Rui } 313e059c184SHuang Rui 314e059c184SHuang Rui DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); 315e059c184SHuang Rui 316e059c184SHuang Rui static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 317e059c184SHuang Rui { 318e059c184SHuang Rui return static_call(amd_pstate_init_perf)(cpudata); 319e059c184SHuang Rui } 320e059c184SHuang Rui 321e059c184SHuang Rui static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, 322ec437d71SHuang Rui u32 des_perf, u32 max_perf, bool fast_switch) 323ec437d71SHuang Rui { 324ec437d71SHuang Rui if (fast_switch) 325ec437d71SHuang Rui wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); 326ec437d71SHuang Rui else 327ec437d71SHuang Rui wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 328ec437d71SHuang Rui READ_ONCE(cpudata->cppc_req_cached)); 329ec437d71SHuang Rui } 330ec437d71SHuang Rui 331e059c184SHuang Rui static void cppc_update_perf(struct amd_cpudata *cpudata, 332e059c184SHuang Rui u32 min_perf, u32 des_perf, 333e059c184SHuang Rui u32 max_perf, bool fast_switch) 334e059c184SHuang Rui { 335e059c184SHuang Rui struct cppc_perf_ctrls perf_ctrls; 336e059c184SHuang Rui 337e059c184SHuang Rui perf_ctrls.max_perf = max_perf; 338e059c184SHuang Rui perf_ctrls.min_perf = min_perf; 339e059c184SHuang Rui perf_ctrls.desired_perf = des_perf; 340e059c184SHuang Rui 341e059c184SHuang Rui cppc_set_perf(cpudata->cpu, &perf_ctrls); 342e059c184SHuang Rui } 343e059c184SHuang Rui 344e059c184SHuang Rui DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); 345e059c184SHuang Rui 346e059c184SHuang Rui static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, 347e059c184SHuang Rui u32 min_perf, u32 des_perf, 348e059c184SHuang Rui u32 max_perf, bool fast_switch) 349e059c184SHuang Rui { 350e059c184SHuang Rui static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, 351e059c184SHuang Rui max_perf, fast_switch); 352e059c184SHuang Rui } 353e059c184SHuang Rui 35423c296fbSJinzhou Su static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 35523c296fbSJinzhou Su { 35623c296fbSJinzhou Su u64 aperf, mperf, tsc; 35723c296fbSJinzhou Su unsigned long flags; 35823c296fbSJinzhou Su 35923c296fbSJinzhou Su local_irq_save(flags); 36023c296fbSJinzhou Su rdmsrl(MSR_IA32_APERF, aperf); 36123c296fbSJinzhou Su rdmsrl(MSR_IA32_MPERF, mperf); 36223c296fbSJinzhou Su tsc = rdtsc(); 36323c296fbSJinzhou Su 36423c296fbSJinzhou Su if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 36523c296fbSJinzhou Su local_irq_restore(flags); 36623c296fbSJinzhou Su return false; 36723c296fbSJinzhou Su } 36823c296fbSJinzhou Su 36923c296fbSJinzhou Su local_irq_restore(flags); 37023c296fbSJinzhou Su 37123c296fbSJinzhou Su cpudata->cur.aperf = aperf; 37223c296fbSJinzhou Su cpudata->cur.mperf = mperf; 37323c296fbSJinzhou Su cpudata->cur.tsc = tsc; 37423c296fbSJinzhou Su cpudata->cur.aperf -= cpudata->prev.aperf; 37523c296fbSJinzhou Su cpudata->cur.mperf -= cpudata->prev.mperf; 37623c296fbSJinzhou Su cpudata->cur.tsc -= cpudata->prev.tsc; 37723c296fbSJinzhou Su 37823c296fbSJinzhou Su cpudata->prev.aperf = aperf; 37923c296fbSJinzhou Su cpudata->prev.mperf = mperf; 38023c296fbSJinzhou Su cpudata->prev.tsc = tsc; 38123c296fbSJinzhou Su 38223c296fbSJinzhou Su cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 38323c296fbSJinzhou Su 38423c296fbSJinzhou Su return true; 38523c296fbSJinzhou Su } 38623c296fbSJinzhou Su 387ec437d71SHuang Rui static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, 388ec437d71SHuang Rui u32 des_perf, u32 max_perf, bool fast_switch) 389ec437d71SHuang Rui { 390ec437d71SHuang Rui u64 prev = READ_ONCE(cpudata->cppc_req_cached); 391ec437d71SHuang Rui u64 value = prev; 392ec437d71SHuang Rui 3930e9a8638SPerry Yuan des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 394ec437d71SHuang Rui value &= ~AMD_CPPC_MIN_PERF(~0L); 395ec437d71SHuang Rui value |= AMD_CPPC_MIN_PERF(min_perf); 396ec437d71SHuang Rui 397ec437d71SHuang Rui value &= ~AMD_CPPC_DES_PERF(~0L); 398ec437d71SHuang Rui value |= AMD_CPPC_DES_PERF(des_perf); 399ec437d71SHuang Rui 400ec437d71SHuang Rui value &= ~AMD_CPPC_MAX_PERF(~0L); 401ec437d71SHuang Rui value |= AMD_CPPC_MAX_PERF(max_perf); 402ec437d71SHuang Rui 40323c296fbSJinzhou Su if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 40423c296fbSJinzhou Su trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 40523c296fbSJinzhou Su cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 40660e10f89SHuang Rui cpudata->cpu, (value != prev), fast_switch); 40723c296fbSJinzhou Su } 40860e10f89SHuang Rui 409ec437d71SHuang Rui if (value == prev) 410ec437d71SHuang Rui return; 411ec437d71SHuang Rui 412ec437d71SHuang Rui WRITE_ONCE(cpudata->cppc_req_cached, value); 413ec437d71SHuang Rui 414ec437d71SHuang Rui amd_pstate_update_perf(cpudata, min_perf, des_perf, 415ec437d71SHuang Rui max_perf, fast_switch); 416ec437d71SHuang Rui } 417ec437d71SHuang Rui 418ec437d71SHuang Rui static int amd_pstate_verify(struct cpufreq_policy_data *policy) 419ec437d71SHuang Rui { 420ec437d71SHuang Rui cpufreq_verify_within_cpu_limits(policy); 421ec437d71SHuang Rui 422ec437d71SHuang Rui return 0; 423ec437d71SHuang Rui } 424ec437d71SHuang Rui 425ec437d71SHuang Rui static int amd_pstate_target(struct cpufreq_policy *policy, 426ec437d71SHuang Rui unsigned int target_freq, 427ec437d71SHuang Rui unsigned int relation) 428ec437d71SHuang Rui { 429ec437d71SHuang Rui struct cpufreq_freqs freqs; 430ec437d71SHuang Rui struct amd_cpudata *cpudata = policy->driver_data; 431ec437d71SHuang Rui unsigned long max_perf, min_perf, des_perf, cap_perf; 432ec437d71SHuang Rui 433ec437d71SHuang Rui if (!cpudata->max_freq) 434ec437d71SHuang Rui return -ENODEV; 435ec437d71SHuang Rui 436ec437d71SHuang Rui cap_perf = READ_ONCE(cpudata->highest_perf); 437b185c505SPerry Yuan min_perf = READ_ONCE(cpudata->lowest_perf); 438ec437d71SHuang Rui max_perf = cap_perf; 439ec437d71SHuang Rui 440ec437d71SHuang Rui freqs.old = policy->cur; 441ec437d71SHuang Rui freqs.new = target_freq; 442ec437d71SHuang Rui 443ec437d71SHuang Rui des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, 444ec437d71SHuang Rui cpudata->max_freq); 445ec437d71SHuang Rui 446ec437d71SHuang Rui cpufreq_freq_transition_begin(policy, &freqs); 447ec437d71SHuang Rui amd_pstate_update(cpudata, min_perf, des_perf, 448ec437d71SHuang Rui max_perf, false); 449ec437d71SHuang Rui cpufreq_freq_transition_end(policy, &freqs, false); 450ec437d71SHuang Rui 451ec437d71SHuang Rui return 0; 452ec437d71SHuang Rui } 453ec437d71SHuang Rui 4541d215f03SHuang Rui static void amd_pstate_adjust_perf(unsigned int cpu, 4551d215f03SHuang Rui unsigned long _min_perf, 4561d215f03SHuang Rui unsigned long target_perf, 4571d215f03SHuang Rui unsigned long capacity) 4581d215f03SHuang Rui { 4591d215f03SHuang Rui unsigned long max_perf, min_perf, des_perf, 4601d215f03SHuang Rui cap_perf, lowest_nonlinear_perf; 4611d215f03SHuang Rui struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 4621d215f03SHuang Rui struct amd_cpudata *cpudata = policy->driver_data; 4631d215f03SHuang Rui 4641d215f03SHuang Rui cap_perf = READ_ONCE(cpudata->highest_perf); 4651d215f03SHuang Rui lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); 4661d215f03SHuang Rui 4671d215f03SHuang Rui des_perf = cap_perf; 4681d215f03SHuang Rui if (target_perf < capacity) 4691d215f03SHuang Rui des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 4701d215f03SHuang Rui 4711d215f03SHuang Rui min_perf = READ_ONCE(cpudata->highest_perf); 4721d215f03SHuang Rui if (_min_perf < capacity) 4731d215f03SHuang Rui min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 4741d215f03SHuang Rui 4751d215f03SHuang Rui if (min_perf < lowest_nonlinear_perf) 4761d215f03SHuang Rui min_perf = lowest_nonlinear_perf; 4771d215f03SHuang Rui 4781d215f03SHuang Rui max_perf = cap_perf; 4791d215f03SHuang Rui if (max_perf < min_perf) 4801d215f03SHuang Rui max_perf = min_perf; 4811d215f03SHuang Rui 4821d215f03SHuang Rui amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); 4834f3085f8SPerry Yuan cpufreq_cpu_put(policy); 4841d215f03SHuang Rui } 4851d215f03SHuang Rui 486ec437d71SHuang Rui static int amd_get_min_freq(struct amd_cpudata *cpudata) 487ec437d71SHuang Rui { 488ec437d71SHuang Rui struct cppc_perf_caps cppc_perf; 489ec437d71SHuang Rui 490ec437d71SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 491ec437d71SHuang Rui if (ret) 492ec437d71SHuang Rui return ret; 493ec437d71SHuang Rui 494ec437d71SHuang Rui /* Switch to khz */ 495ec437d71SHuang Rui return cppc_perf.lowest_freq * 1000; 496ec437d71SHuang Rui } 497ec437d71SHuang Rui 498ec437d71SHuang Rui static int amd_get_max_freq(struct amd_cpudata *cpudata) 499ec437d71SHuang Rui { 500ec437d71SHuang Rui struct cppc_perf_caps cppc_perf; 501ec437d71SHuang Rui u32 max_perf, max_freq, nominal_freq, nominal_perf; 502ec437d71SHuang Rui u64 boost_ratio; 503ec437d71SHuang Rui 504ec437d71SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 505ec437d71SHuang Rui if (ret) 506ec437d71SHuang Rui return ret; 507ec437d71SHuang Rui 508ec437d71SHuang Rui nominal_freq = cppc_perf.nominal_freq; 509ec437d71SHuang Rui nominal_perf = READ_ONCE(cpudata->nominal_perf); 510ec437d71SHuang Rui max_perf = READ_ONCE(cpudata->highest_perf); 511ec437d71SHuang Rui 512ec437d71SHuang Rui boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT, 513ec437d71SHuang Rui nominal_perf); 514ec437d71SHuang Rui 515ec437d71SHuang Rui max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT; 516ec437d71SHuang Rui 517ec437d71SHuang Rui /* Switch to khz */ 518ec437d71SHuang Rui return max_freq * 1000; 519ec437d71SHuang Rui } 520ec437d71SHuang Rui 521ec437d71SHuang Rui static int amd_get_nominal_freq(struct amd_cpudata *cpudata) 522ec437d71SHuang Rui { 523ec437d71SHuang Rui struct cppc_perf_caps cppc_perf; 524ec437d71SHuang Rui 525ec437d71SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 526ec437d71SHuang Rui if (ret) 527ec437d71SHuang Rui return ret; 528ec437d71SHuang Rui 529ec437d71SHuang Rui /* Switch to khz */ 530ec437d71SHuang Rui return cppc_perf.nominal_freq * 1000; 531ec437d71SHuang Rui } 532ec437d71SHuang Rui 533ec437d71SHuang Rui static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) 534ec437d71SHuang Rui { 535ec437d71SHuang Rui struct cppc_perf_caps cppc_perf; 536ec437d71SHuang Rui u32 lowest_nonlinear_freq, lowest_nonlinear_perf, 537ec437d71SHuang Rui nominal_freq, nominal_perf; 538ec437d71SHuang Rui u64 lowest_nonlinear_ratio; 539ec437d71SHuang Rui 540ec437d71SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 541ec437d71SHuang Rui if (ret) 542ec437d71SHuang Rui return ret; 543ec437d71SHuang Rui 544ec437d71SHuang Rui nominal_freq = cppc_perf.nominal_freq; 545ec437d71SHuang Rui nominal_perf = READ_ONCE(cpudata->nominal_perf); 546ec437d71SHuang Rui 547ec437d71SHuang Rui lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 548ec437d71SHuang Rui 549ec437d71SHuang Rui lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, 550ec437d71SHuang Rui nominal_perf); 551ec437d71SHuang Rui 552ec437d71SHuang Rui lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT; 553ec437d71SHuang Rui 554ec437d71SHuang Rui /* Switch to khz */ 555ec437d71SHuang Rui return lowest_nonlinear_freq * 1000; 556ec437d71SHuang Rui } 557ec437d71SHuang Rui 55841271016SHuang Rui static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 55941271016SHuang Rui { 56041271016SHuang Rui struct amd_cpudata *cpudata = policy->driver_data; 56141271016SHuang Rui int ret; 56241271016SHuang Rui 56341271016SHuang Rui if (!cpudata->boost_supported) { 56441271016SHuang Rui pr_err("Boost mode is not supported by this processor or SBIOS\n"); 56541271016SHuang Rui return -EINVAL; 56641271016SHuang Rui } 56741271016SHuang Rui 56841271016SHuang Rui if (state) 56941271016SHuang Rui policy->cpuinfo.max_freq = cpudata->max_freq; 57041271016SHuang Rui else 57141271016SHuang Rui policy->cpuinfo.max_freq = cpudata->nominal_freq; 57241271016SHuang Rui 57341271016SHuang Rui policy->max = policy->cpuinfo.max_freq; 57441271016SHuang Rui 57541271016SHuang Rui ret = freq_qos_update_request(&cpudata->req[1], 57641271016SHuang Rui policy->cpuinfo.max_freq); 57741271016SHuang Rui if (ret < 0) 57841271016SHuang Rui return ret; 57941271016SHuang Rui 58041271016SHuang Rui return 0; 58141271016SHuang Rui } 58241271016SHuang Rui 58341271016SHuang Rui static void amd_pstate_boost_init(struct amd_cpudata *cpudata) 58441271016SHuang Rui { 58541271016SHuang Rui u32 highest_perf, nominal_perf; 58641271016SHuang Rui 58741271016SHuang Rui highest_perf = READ_ONCE(cpudata->highest_perf); 58841271016SHuang Rui nominal_perf = READ_ONCE(cpudata->nominal_perf); 58941271016SHuang Rui 59041271016SHuang Rui if (highest_perf <= nominal_perf) 59141271016SHuang Rui return; 59241271016SHuang Rui 59341271016SHuang Rui cpudata->boost_supported = true; 594ffa5096aSPerry Yuan current_pstate_driver->boost_enabled = true; 59541271016SHuang Rui } 59641271016SHuang Rui 597919f4557SWyes Karny static void amd_perf_ctl_reset(unsigned int cpu) 598919f4557SWyes Karny { 599919f4557SWyes Karny wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 600919f4557SWyes Karny } 601919f4557SWyes Karny 602ec437d71SHuang Rui static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 603ec437d71SHuang Rui { 604ec437d71SHuang Rui int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 605ec437d71SHuang Rui struct device *dev; 606ec437d71SHuang Rui struct amd_cpudata *cpudata; 607ec437d71SHuang Rui 608919f4557SWyes Karny /* 609919f4557SWyes Karny * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 610919f4557SWyes Karny * which is ideal for initialization process. 611919f4557SWyes Karny */ 612919f4557SWyes Karny amd_perf_ctl_reset(policy->cpu); 613ec437d71SHuang Rui dev = get_cpu_device(policy->cpu); 614ec437d71SHuang Rui if (!dev) 615ec437d71SHuang Rui return -ENODEV; 616ec437d71SHuang Rui 617ec437d71SHuang Rui cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 618ec437d71SHuang Rui if (!cpudata) 619ec437d71SHuang Rui return -ENOMEM; 620ec437d71SHuang Rui 621ec437d71SHuang Rui cpudata->cpu = policy->cpu; 622ec437d71SHuang Rui 623ec437d71SHuang Rui ret = amd_pstate_init_perf(cpudata); 624ec437d71SHuang Rui if (ret) 62541271016SHuang Rui goto free_cpudata1; 626ec437d71SHuang Rui 627ec437d71SHuang Rui min_freq = amd_get_min_freq(cpudata); 628ec437d71SHuang Rui max_freq = amd_get_max_freq(cpudata); 629ec437d71SHuang Rui nominal_freq = amd_get_nominal_freq(cpudata); 630ec437d71SHuang Rui lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 631ec437d71SHuang Rui 632ec437d71SHuang Rui if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 633ec437d71SHuang Rui dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 634ec437d71SHuang Rui min_freq, max_freq); 635ec437d71SHuang Rui ret = -EINVAL; 63641271016SHuang Rui goto free_cpudata1; 637ec437d71SHuang Rui } 638ec437d71SHuang Rui 639ec437d71SHuang Rui policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; 640ec437d71SHuang Rui policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; 641ec437d71SHuang Rui 642ec437d71SHuang Rui policy->min = min_freq; 643ec437d71SHuang Rui policy->max = max_freq; 644ec437d71SHuang Rui 645ec437d71SHuang Rui policy->cpuinfo.min_freq = min_freq; 646ec437d71SHuang Rui policy->cpuinfo.max_freq = max_freq; 647ec437d71SHuang Rui 648ec437d71SHuang Rui /* It will be updated by governor */ 649ec437d71SHuang Rui policy->cur = policy->cpuinfo.min_freq; 650ec437d71SHuang Rui 651e059c184SHuang Rui if (boot_cpu_has(X86_FEATURE_CPPC)) 6521d215f03SHuang Rui policy->fast_switch_possible = true; 6531d215f03SHuang Rui 65441271016SHuang Rui ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 65541271016SHuang Rui FREQ_QOS_MIN, policy->cpuinfo.min_freq); 65641271016SHuang Rui if (ret < 0) { 65741271016SHuang Rui dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 65841271016SHuang Rui goto free_cpudata1; 65941271016SHuang Rui } 66041271016SHuang Rui 66141271016SHuang Rui ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 66241271016SHuang Rui FREQ_QOS_MAX, policy->cpuinfo.max_freq); 66341271016SHuang Rui if (ret < 0) { 66441271016SHuang Rui dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 66541271016SHuang Rui goto free_cpudata2; 66641271016SHuang Rui } 66741271016SHuang Rui 668ec437d71SHuang Rui /* Initial processor data capability frequencies */ 669ec437d71SHuang Rui cpudata->max_freq = max_freq; 670ec437d71SHuang Rui cpudata->min_freq = min_freq; 671ec437d71SHuang Rui cpudata->nominal_freq = nominal_freq; 672ec437d71SHuang Rui cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 673ec437d71SHuang Rui 674ec437d71SHuang Rui policy->driver_data = cpudata; 675ec437d71SHuang Rui 67641271016SHuang Rui amd_pstate_boost_init(cpudata); 677abd61c08SPerry Yuan if (!current_pstate_driver->adjust_perf) 678abd61c08SPerry Yuan current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 67941271016SHuang Rui 680ec437d71SHuang Rui return 0; 681ec437d71SHuang Rui 68241271016SHuang Rui free_cpudata2: 68341271016SHuang Rui freq_qos_remove_request(&cpudata->req[0]); 68441271016SHuang Rui free_cpudata1: 685ec437d71SHuang Rui kfree(cpudata); 686ec437d71SHuang Rui return ret; 687ec437d71SHuang Rui } 688ec437d71SHuang Rui 689ec437d71SHuang Rui static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) 690ec437d71SHuang Rui { 6914f59540cSPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 692ec437d71SHuang Rui 69341271016SHuang Rui freq_qos_remove_request(&cpudata->req[1]); 69441271016SHuang Rui freq_qos_remove_request(&cpudata->req[0]); 695ec437d71SHuang Rui kfree(cpudata); 696ec437d71SHuang Rui 697ec437d71SHuang Rui return 0; 698ec437d71SHuang Rui } 699ec437d71SHuang Rui 700b376471fSJinzhou Su static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) 701b376471fSJinzhou Su { 702b376471fSJinzhou Su int ret; 703b376471fSJinzhou Su 704b376471fSJinzhou Su ret = amd_pstate_enable(true); 705b376471fSJinzhou Su if (ret) 706b376471fSJinzhou Su pr_err("failed to enable amd-pstate during resume, return %d\n", ret); 707b376471fSJinzhou Su 708b376471fSJinzhou Su return ret; 709b376471fSJinzhou Su } 710b376471fSJinzhou Su 711b376471fSJinzhou Su static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) 712b376471fSJinzhou Su { 713b376471fSJinzhou Su int ret; 714b376471fSJinzhou Su 715b376471fSJinzhou Su ret = amd_pstate_enable(false); 716b376471fSJinzhou Su if (ret) 717b376471fSJinzhou Su pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); 718b376471fSJinzhou Su 719b376471fSJinzhou Su return ret; 720b376471fSJinzhou Su } 721b376471fSJinzhou Su 722ec4e3326SHuang Rui /* Sysfs attributes */ 723ec4e3326SHuang Rui 724ec4e3326SHuang Rui /* 725ec4e3326SHuang Rui * This frequency is to indicate the maximum hardware frequency. 726ec4e3326SHuang Rui * If boost is not active but supported, the frequency will be larger than the 727ec4e3326SHuang Rui * one in cpuinfo. 728ec4e3326SHuang Rui */ 729ec4e3326SHuang Rui static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 730ec4e3326SHuang Rui char *buf) 731ec4e3326SHuang Rui { 732ec4e3326SHuang Rui int max_freq; 7334f59540cSPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 734ec4e3326SHuang Rui 735ec4e3326SHuang Rui max_freq = amd_get_max_freq(cpudata); 736ec4e3326SHuang Rui if (max_freq < 0) 737ec4e3326SHuang Rui return max_freq; 738ec4e3326SHuang Rui 7393ec32b6dSPerry Yuan return sysfs_emit(buf, "%u\n", max_freq); 740ec4e3326SHuang Rui } 741ec4e3326SHuang Rui 742ec4e3326SHuang Rui static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 743ec4e3326SHuang Rui char *buf) 744ec4e3326SHuang Rui { 745ec4e3326SHuang Rui int freq; 7464f59540cSPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 747ec4e3326SHuang Rui 748ec4e3326SHuang Rui freq = amd_get_lowest_nonlinear_freq(cpudata); 749ec4e3326SHuang Rui if (freq < 0) 750ec4e3326SHuang Rui return freq; 751ec4e3326SHuang Rui 7523ec32b6dSPerry Yuan return sysfs_emit(buf, "%u\n", freq); 753ec4e3326SHuang Rui } 754ec4e3326SHuang Rui 7553ad7fde1SHuang Rui /* 7563ad7fde1SHuang Rui * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 7573ad7fde1SHuang Rui * need to expose it to sysfs. 7583ad7fde1SHuang Rui */ 7593ad7fde1SHuang Rui static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 7603ad7fde1SHuang Rui char *buf) 7613ad7fde1SHuang Rui { 7623ad7fde1SHuang Rui u32 perf; 7633ad7fde1SHuang Rui struct amd_cpudata *cpudata = policy->driver_data; 7643ad7fde1SHuang Rui 7653ad7fde1SHuang Rui perf = READ_ONCE(cpudata->highest_perf); 7663ad7fde1SHuang Rui 7673ec32b6dSPerry Yuan return sysfs_emit(buf, "%u\n", perf); 7683ad7fde1SHuang Rui } 7693ad7fde1SHuang Rui 770ffa5096aSPerry Yuan static ssize_t show_energy_performance_available_preferences( 771ffa5096aSPerry Yuan struct cpufreq_policy *policy, char *buf) 772ffa5096aSPerry Yuan { 773ffa5096aSPerry Yuan int i = 0; 774ffa5096aSPerry Yuan int offset = 0; 775ffa5096aSPerry Yuan 776ffa5096aSPerry Yuan while (energy_perf_strings[i] != NULL) 777ffa5096aSPerry Yuan offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); 778ffa5096aSPerry Yuan 779ffa5096aSPerry Yuan sysfs_emit_at(buf, offset, "\n"); 780ffa5096aSPerry Yuan 781ffa5096aSPerry Yuan return offset; 782ffa5096aSPerry Yuan } 783ffa5096aSPerry Yuan 784ffa5096aSPerry Yuan static ssize_t store_energy_performance_preference( 785ffa5096aSPerry Yuan struct cpufreq_policy *policy, const char *buf, size_t count) 786ffa5096aSPerry Yuan { 787ffa5096aSPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 788ffa5096aSPerry Yuan char str_preference[21]; 789ffa5096aSPerry Yuan ssize_t ret; 790ffa5096aSPerry Yuan 791ffa5096aSPerry Yuan ret = sscanf(buf, "%20s", str_preference); 792ffa5096aSPerry Yuan if (ret != 1) 793ffa5096aSPerry Yuan return -EINVAL; 794ffa5096aSPerry Yuan 795ffa5096aSPerry Yuan ret = match_string(energy_perf_strings, -1, str_preference); 796ffa5096aSPerry Yuan if (ret < 0) 797ffa5096aSPerry Yuan return -EINVAL; 798ffa5096aSPerry Yuan 799ffa5096aSPerry Yuan mutex_lock(&amd_pstate_limits_lock); 800ffa5096aSPerry Yuan ret = amd_pstate_set_energy_pref_index(cpudata, ret); 801ffa5096aSPerry Yuan mutex_unlock(&amd_pstate_limits_lock); 802ffa5096aSPerry Yuan 803ffa5096aSPerry Yuan return ret ?: count; 804ffa5096aSPerry Yuan } 805ffa5096aSPerry Yuan 806ffa5096aSPerry Yuan static ssize_t show_energy_performance_preference( 807ffa5096aSPerry Yuan struct cpufreq_policy *policy, char *buf) 808ffa5096aSPerry Yuan { 809ffa5096aSPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 810ffa5096aSPerry Yuan int preference; 811ffa5096aSPerry Yuan 812ffa5096aSPerry Yuan preference = amd_pstate_get_energy_pref_index(cpudata); 813ffa5096aSPerry Yuan if (preference < 0) 814ffa5096aSPerry Yuan return preference; 815ffa5096aSPerry Yuan 816ffa5096aSPerry Yuan return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); 817ffa5096aSPerry Yuan } 818ffa5096aSPerry Yuan 819abd61c08SPerry Yuan static ssize_t amd_pstate_show_status(char *buf) 820abd61c08SPerry Yuan { 821abd61c08SPerry Yuan if (!current_pstate_driver) 822abd61c08SPerry Yuan return sysfs_emit(buf, "disable\n"); 823abd61c08SPerry Yuan 824abd61c08SPerry Yuan return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); 825abd61c08SPerry Yuan } 826abd61c08SPerry Yuan 827abd61c08SPerry Yuan static void amd_pstate_driver_cleanup(void) 828abd61c08SPerry Yuan { 829abd61c08SPerry Yuan current_pstate_driver = NULL; 830abd61c08SPerry Yuan } 831abd61c08SPerry Yuan 832abd61c08SPerry Yuan static int amd_pstate_update_status(const char *buf, size_t size) 833abd61c08SPerry Yuan { 834*dd329e1eSUwe Kleine-König int ret = 0; 835abd61c08SPerry Yuan int mode_idx; 836abd61c08SPerry Yuan 837abd61c08SPerry Yuan if (size > 7 || size < 6) 838abd61c08SPerry Yuan return -EINVAL; 839abd61c08SPerry Yuan mode_idx = get_mode_idx_from_str(buf, size); 840abd61c08SPerry Yuan 841abd61c08SPerry Yuan switch(mode_idx) { 842abd61c08SPerry Yuan case AMD_PSTATE_DISABLE: 843abd61c08SPerry Yuan if (!current_pstate_driver) 844abd61c08SPerry Yuan return -EINVAL; 845abd61c08SPerry Yuan if (cppc_state == AMD_PSTATE_ACTIVE) 846abd61c08SPerry Yuan return -EBUSY; 847*dd329e1eSUwe Kleine-König cpufreq_unregister_driver(current_pstate_driver); 848abd61c08SPerry Yuan amd_pstate_driver_cleanup(); 849abd61c08SPerry Yuan break; 850abd61c08SPerry Yuan case AMD_PSTATE_PASSIVE: 851abd61c08SPerry Yuan if (current_pstate_driver) { 852abd61c08SPerry Yuan if (current_pstate_driver == &amd_pstate_driver) 853abd61c08SPerry Yuan return 0; 854abd61c08SPerry Yuan cpufreq_unregister_driver(current_pstate_driver); 855abd61c08SPerry Yuan cppc_state = AMD_PSTATE_PASSIVE; 856abd61c08SPerry Yuan current_pstate_driver = &amd_pstate_driver; 857abd61c08SPerry Yuan } 858abd61c08SPerry Yuan 859abd61c08SPerry Yuan ret = cpufreq_register_driver(current_pstate_driver); 860abd61c08SPerry Yuan break; 861abd61c08SPerry Yuan case AMD_PSTATE_ACTIVE: 862abd61c08SPerry Yuan if (current_pstate_driver) { 863abd61c08SPerry Yuan if (current_pstate_driver == &amd_pstate_epp_driver) 864abd61c08SPerry Yuan return 0; 865abd61c08SPerry Yuan cpufreq_unregister_driver(current_pstate_driver); 866abd61c08SPerry Yuan current_pstate_driver = &amd_pstate_epp_driver; 867abd61c08SPerry Yuan cppc_state = AMD_PSTATE_ACTIVE; 868abd61c08SPerry Yuan } 869abd61c08SPerry Yuan 870abd61c08SPerry Yuan ret = cpufreq_register_driver(current_pstate_driver); 871abd61c08SPerry Yuan break; 872abd61c08SPerry Yuan default: 873abd61c08SPerry Yuan ret = -EINVAL; 874abd61c08SPerry Yuan break; 875abd61c08SPerry Yuan } 876abd61c08SPerry Yuan 877abd61c08SPerry Yuan return ret; 878abd61c08SPerry Yuan } 879abd61c08SPerry Yuan 880abd61c08SPerry Yuan static ssize_t show_status(struct kobject *kobj, 881abd61c08SPerry Yuan struct kobj_attribute *attr, char *buf) 882abd61c08SPerry Yuan { 883abd61c08SPerry Yuan ssize_t ret; 884abd61c08SPerry Yuan 885abd61c08SPerry Yuan mutex_lock(&amd_pstate_driver_lock); 886abd61c08SPerry Yuan ret = amd_pstate_show_status(buf); 887abd61c08SPerry Yuan mutex_unlock(&amd_pstate_driver_lock); 888abd61c08SPerry Yuan 889abd61c08SPerry Yuan return ret; 890abd61c08SPerry Yuan } 891abd61c08SPerry Yuan 892abd61c08SPerry Yuan static ssize_t store_status(struct kobject *a, struct kobj_attribute *b, 893abd61c08SPerry Yuan const char *buf, size_t count) 894abd61c08SPerry Yuan { 895abd61c08SPerry Yuan char *p = memchr(buf, '\n', count); 896abd61c08SPerry Yuan int ret; 897abd61c08SPerry Yuan 898abd61c08SPerry Yuan mutex_lock(&amd_pstate_driver_lock); 899abd61c08SPerry Yuan ret = amd_pstate_update_status(buf, p ? p - buf : count); 900abd61c08SPerry Yuan mutex_unlock(&amd_pstate_driver_lock); 901abd61c08SPerry Yuan 902abd61c08SPerry Yuan return ret < 0 ? ret : count; 903abd61c08SPerry Yuan } 904abd61c08SPerry Yuan 905ec4e3326SHuang Rui cpufreq_freq_attr_ro(amd_pstate_max_freq); 906ec4e3326SHuang Rui cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 907ec4e3326SHuang Rui 9083ad7fde1SHuang Rui cpufreq_freq_attr_ro(amd_pstate_highest_perf); 909ffa5096aSPerry Yuan cpufreq_freq_attr_rw(energy_performance_preference); 910ffa5096aSPerry Yuan cpufreq_freq_attr_ro(energy_performance_available_preferences); 911abd61c08SPerry Yuan define_one_global_rw(status); 9123ad7fde1SHuang Rui 913ec4e3326SHuang Rui static struct freq_attr *amd_pstate_attr[] = { 914ec4e3326SHuang Rui &amd_pstate_max_freq, 915ec4e3326SHuang Rui &amd_pstate_lowest_nonlinear_freq, 9163ad7fde1SHuang Rui &amd_pstate_highest_perf, 917ec4e3326SHuang Rui NULL, 918ec4e3326SHuang Rui }; 919ec4e3326SHuang Rui 920ffa5096aSPerry Yuan static struct freq_attr *amd_pstate_epp_attr[] = { 921ffa5096aSPerry Yuan &amd_pstate_max_freq, 922ffa5096aSPerry Yuan &amd_pstate_lowest_nonlinear_freq, 923ffa5096aSPerry Yuan &amd_pstate_highest_perf, 924ffa5096aSPerry Yuan &energy_performance_preference, 925ffa5096aSPerry Yuan &energy_performance_available_preferences, 926ffa5096aSPerry Yuan NULL, 927ffa5096aSPerry Yuan }; 928ffa5096aSPerry Yuan 929abd61c08SPerry Yuan static struct attribute *pstate_global_attributes[] = { 930abd61c08SPerry Yuan &status.attr, 931abd61c08SPerry Yuan NULL 932abd61c08SPerry Yuan }; 933abd61c08SPerry Yuan 934abd61c08SPerry Yuan static const struct attribute_group amd_pstate_global_attr_group = { 935abd61c08SPerry Yuan .attrs = pstate_global_attributes, 936abd61c08SPerry Yuan }; 937abd61c08SPerry Yuan 938ffa5096aSPerry Yuan static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) 939ffa5096aSPerry Yuan { 940ffa5096aSPerry Yuan int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 941ffa5096aSPerry Yuan struct amd_cpudata *cpudata; 942ffa5096aSPerry Yuan struct device *dev; 943ffa5096aSPerry Yuan int rc; 944ffa5096aSPerry Yuan u64 value; 945ffa5096aSPerry Yuan 946ffa5096aSPerry Yuan /* 947ffa5096aSPerry Yuan * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 948ffa5096aSPerry Yuan * which is ideal for initialization process. 949ffa5096aSPerry Yuan */ 950ffa5096aSPerry Yuan amd_perf_ctl_reset(policy->cpu); 951ffa5096aSPerry Yuan dev = get_cpu_device(policy->cpu); 952ffa5096aSPerry Yuan if (!dev) 953ffa5096aSPerry Yuan goto free_cpudata1; 954ffa5096aSPerry Yuan 955ffa5096aSPerry Yuan cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 956ffa5096aSPerry Yuan if (!cpudata) 957ffa5096aSPerry Yuan return -ENOMEM; 958ffa5096aSPerry Yuan 959ffa5096aSPerry Yuan cpudata->cpu = policy->cpu; 960ffa5096aSPerry Yuan cpudata->epp_policy = 0; 961ffa5096aSPerry Yuan 962ffa5096aSPerry Yuan rc = amd_pstate_init_perf(cpudata); 963ffa5096aSPerry Yuan if (rc) 964ffa5096aSPerry Yuan goto free_cpudata1; 965ffa5096aSPerry Yuan 966ffa5096aSPerry Yuan min_freq = amd_get_min_freq(cpudata); 967ffa5096aSPerry Yuan max_freq = amd_get_max_freq(cpudata); 968ffa5096aSPerry Yuan nominal_freq = amd_get_nominal_freq(cpudata); 969ffa5096aSPerry Yuan lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 970ffa5096aSPerry Yuan if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 971ffa5096aSPerry Yuan dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 972ffa5096aSPerry Yuan min_freq, max_freq); 973ffa5096aSPerry Yuan ret = -EINVAL; 974ffa5096aSPerry Yuan goto free_cpudata1; 975ffa5096aSPerry Yuan } 976ffa5096aSPerry Yuan 977ffa5096aSPerry Yuan policy->cpuinfo.min_freq = min_freq; 978ffa5096aSPerry Yuan policy->cpuinfo.max_freq = max_freq; 979ffa5096aSPerry Yuan /* It will be updated by governor */ 980ffa5096aSPerry Yuan policy->cur = policy->cpuinfo.min_freq; 981ffa5096aSPerry Yuan 982ffa5096aSPerry Yuan /* Initial processor data capability frequencies */ 983ffa5096aSPerry Yuan cpudata->max_freq = max_freq; 984ffa5096aSPerry Yuan cpudata->min_freq = min_freq; 985ffa5096aSPerry Yuan cpudata->nominal_freq = nominal_freq; 986ffa5096aSPerry Yuan cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 987ffa5096aSPerry Yuan 988ffa5096aSPerry Yuan policy->driver_data = cpudata; 989ffa5096aSPerry Yuan 990ffa5096aSPerry Yuan cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); 991ffa5096aSPerry Yuan 992ffa5096aSPerry Yuan policy->min = policy->cpuinfo.min_freq; 993ffa5096aSPerry Yuan policy->max = policy->cpuinfo.max_freq; 994ffa5096aSPerry Yuan 995ffa5096aSPerry Yuan /* 996ffa5096aSPerry Yuan * Set the policy to powersave to provide a valid fallback value in case 997ffa5096aSPerry Yuan * the default cpufreq governor is neither powersave nor performance. 998ffa5096aSPerry Yuan */ 999ffa5096aSPerry Yuan policy->policy = CPUFREQ_POLICY_POWERSAVE; 1000ffa5096aSPerry Yuan 1001ffa5096aSPerry Yuan if (boot_cpu_has(X86_FEATURE_CPPC)) { 1002ffa5096aSPerry Yuan policy->fast_switch_possible = true; 1003ffa5096aSPerry Yuan ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); 1004ffa5096aSPerry Yuan if (ret) 1005ffa5096aSPerry Yuan return ret; 1006ffa5096aSPerry Yuan WRITE_ONCE(cpudata->cppc_req_cached, value); 1007ffa5096aSPerry Yuan 1008ffa5096aSPerry Yuan ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); 1009ffa5096aSPerry Yuan if (ret) 1010ffa5096aSPerry Yuan return ret; 1011ffa5096aSPerry Yuan WRITE_ONCE(cpudata->cppc_cap1_cached, value); 1012ffa5096aSPerry Yuan } 1013ffa5096aSPerry Yuan amd_pstate_boost_init(cpudata); 1014ffa5096aSPerry Yuan 1015ffa5096aSPerry Yuan return 0; 1016ffa5096aSPerry Yuan 1017ffa5096aSPerry Yuan free_cpudata1: 1018ffa5096aSPerry Yuan kfree(cpudata); 1019ffa5096aSPerry Yuan return ret; 1020ffa5096aSPerry Yuan } 1021ffa5096aSPerry Yuan 1022ffa5096aSPerry Yuan static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) 1023ffa5096aSPerry Yuan { 1024ffa5096aSPerry Yuan pr_debug("CPU %d exiting\n", policy->cpu); 1025ffa5096aSPerry Yuan policy->fast_switch_possible = false; 1026ffa5096aSPerry Yuan return 0; 1027ffa5096aSPerry Yuan } 1028ffa5096aSPerry Yuan 1029ffa5096aSPerry Yuan static void amd_pstate_epp_init(unsigned int cpu) 1030ffa5096aSPerry Yuan { 1031ffa5096aSPerry Yuan struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 1032ffa5096aSPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 1033ffa5096aSPerry Yuan u32 max_perf, min_perf; 1034ffa5096aSPerry Yuan u64 value; 1035ffa5096aSPerry Yuan s16 epp; 1036ffa5096aSPerry Yuan 1037ffa5096aSPerry Yuan max_perf = READ_ONCE(cpudata->highest_perf); 1038ffa5096aSPerry Yuan min_perf = READ_ONCE(cpudata->lowest_perf); 1039ffa5096aSPerry Yuan 1040ffa5096aSPerry Yuan value = READ_ONCE(cpudata->cppc_req_cached); 1041ffa5096aSPerry Yuan 1042ffa5096aSPerry Yuan if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) 1043ffa5096aSPerry Yuan min_perf = max_perf; 1044ffa5096aSPerry Yuan 1045ffa5096aSPerry Yuan /* Initial min/max values for CPPC Performance Controls Register */ 1046ffa5096aSPerry Yuan value &= ~AMD_CPPC_MIN_PERF(~0L); 1047ffa5096aSPerry Yuan value |= AMD_CPPC_MIN_PERF(min_perf); 1048ffa5096aSPerry Yuan 1049ffa5096aSPerry Yuan value &= ~AMD_CPPC_MAX_PERF(~0L); 1050ffa5096aSPerry Yuan value |= AMD_CPPC_MAX_PERF(max_perf); 1051ffa5096aSPerry Yuan 1052ffa5096aSPerry Yuan /* CPPC EPP feature require to set zero to the desire perf bit */ 1053ffa5096aSPerry Yuan value &= ~AMD_CPPC_DES_PERF(~0L); 1054ffa5096aSPerry Yuan value |= AMD_CPPC_DES_PERF(0); 1055ffa5096aSPerry Yuan 1056ffa5096aSPerry Yuan if (cpudata->epp_policy == cpudata->policy) 1057ffa5096aSPerry Yuan goto skip_epp; 1058ffa5096aSPerry Yuan 1059ffa5096aSPerry Yuan cpudata->epp_policy = cpudata->policy; 1060ffa5096aSPerry Yuan 1061ffa5096aSPerry Yuan if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { 1062ffa5096aSPerry Yuan epp = amd_pstate_get_epp(cpudata, value); 1063ffa5096aSPerry Yuan if (epp < 0) 1064ffa5096aSPerry Yuan goto skip_epp; 1065ffa5096aSPerry Yuan /* force the epp value to be zero for performance policy */ 1066ffa5096aSPerry Yuan epp = 0; 1067ffa5096aSPerry Yuan } else { 1068ffa5096aSPerry Yuan /* Get BIOS pre-defined epp value */ 1069ffa5096aSPerry Yuan epp = amd_pstate_get_epp(cpudata, value); 1070ffa5096aSPerry Yuan if (epp) 1071ffa5096aSPerry Yuan goto skip_epp; 1072ffa5096aSPerry Yuan } 1073ffa5096aSPerry Yuan /* Set initial EPP value */ 1074ffa5096aSPerry Yuan if (boot_cpu_has(X86_FEATURE_CPPC)) { 1075ffa5096aSPerry Yuan value &= ~GENMASK_ULL(31, 24); 1076ffa5096aSPerry Yuan value |= (u64)epp << 24; 1077ffa5096aSPerry Yuan } 1078ffa5096aSPerry Yuan 1079ffa5096aSPerry Yuan skip_epp: 1080ffa5096aSPerry Yuan WRITE_ONCE(cpudata->cppc_req_cached, value); 1081ffa5096aSPerry Yuan amd_pstate_set_epp(cpudata, epp); 1082ffa5096aSPerry Yuan cpufreq_cpu_put(policy); 1083ffa5096aSPerry Yuan } 1084ffa5096aSPerry Yuan 1085ffa5096aSPerry Yuan static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) 1086ffa5096aSPerry Yuan { 1087ffa5096aSPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 1088ffa5096aSPerry Yuan 1089ffa5096aSPerry Yuan if (!policy->cpuinfo.max_freq) 1090ffa5096aSPerry Yuan return -ENODEV; 1091ffa5096aSPerry Yuan 1092ffa5096aSPerry Yuan pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", 1093ffa5096aSPerry Yuan policy->cpuinfo.max_freq, policy->max); 1094ffa5096aSPerry Yuan 1095ffa5096aSPerry Yuan cpudata->policy = policy->policy; 1096ffa5096aSPerry Yuan 1097ffa5096aSPerry Yuan amd_pstate_epp_init(policy->cpu); 1098ffa5096aSPerry Yuan 1099ffa5096aSPerry Yuan return 0; 1100ffa5096aSPerry Yuan } 1101ffa5096aSPerry Yuan 1102d4da12f8SPerry Yuan static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) 1103d4da12f8SPerry Yuan { 1104d4da12f8SPerry Yuan struct cppc_perf_ctrls perf_ctrls; 1105d4da12f8SPerry Yuan u64 value, max_perf; 1106d4da12f8SPerry Yuan int ret; 1107d4da12f8SPerry Yuan 1108d4da12f8SPerry Yuan ret = amd_pstate_enable(true); 1109d4da12f8SPerry Yuan if (ret) 1110d4da12f8SPerry Yuan pr_err("failed to enable amd pstate during resume, return %d\n", ret); 1111d4da12f8SPerry Yuan 1112d4da12f8SPerry Yuan value = READ_ONCE(cpudata->cppc_req_cached); 1113d4da12f8SPerry Yuan max_perf = READ_ONCE(cpudata->highest_perf); 1114d4da12f8SPerry Yuan 1115d4da12f8SPerry Yuan if (boot_cpu_has(X86_FEATURE_CPPC)) { 1116d4da12f8SPerry Yuan wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1117d4da12f8SPerry Yuan } else { 1118d4da12f8SPerry Yuan perf_ctrls.max_perf = max_perf; 1119d4da12f8SPerry Yuan perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); 1120d4da12f8SPerry Yuan cppc_set_perf(cpudata->cpu, &perf_ctrls); 1121d4da12f8SPerry Yuan } 1122d4da12f8SPerry Yuan } 1123d4da12f8SPerry Yuan 1124d4da12f8SPerry Yuan static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) 1125d4da12f8SPerry Yuan { 1126d4da12f8SPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 1127d4da12f8SPerry Yuan 1128d4da12f8SPerry Yuan pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); 1129d4da12f8SPerry Yuan 1130d4da12f8SPerry Yuan if (cppc_state == AMD_PSTATE_ACTIVE) { 1131d4da12f8SPerry Yuan amd_pstate_epp_reenable(cpudata); 1132d4da12f8SPerry Yuan cpudata->suspended = false; 1133d4da12f8SPerry Yuan } 1134d4da12f8SPerry Yuan 1135d4da12f8SPerry Yuan return 0; 1136d4da12f8SPerry Yuan } 1137d4da12f8SPerry Yuan 1138d4da12f8SPerry Yuan static void amd_pstate_epp_offline(struct cpufreq_policy *policy) 1139d4da12f8SPerry Yuan { 1140d4da12f8SPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 1141d4da12f8SPerry Yuan struct cppc_perf_ctrls perf_ctrls; 1142d4da12f8SPerry Yuan int min_perf; 1143d4da12f8SPerry Yuan u64 value; 1144d4da12f8SPerry Yuan 1145d4da12f8SPerry Yuan min_perf = READ_ONCE(cpudata->lowest_perf); 1146d4da12f8SPerry Yuan value = READ_ONCE(cpudata->cppc_req_cached); 1147d4da12f8SPerry Yuan 1148d4da12f8SPerry Yuan mutex_lock(&amd_pstate_limits_lock); 1149d4da12f8SPerry Yuan if (boot_cpu_has(X86_FEATURE_CPPC)) { 1150d4da12f8SPerry Yuan cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; 1151d4da12f8SPerry Yuan 1152d4da12f8SPerry Yuan /* Set max perf same as min perf */ 1153d4da12f8SPerry Yuan value &= ~AMD_CPPC_MAX_PERF(~0L); 1154d4da12f8SPerry Yuan value |= AMD_CPPC_MAX_PERF(min_perf); 1155d4da12f8SPerry Yuan value &= ~AMD_CPPC_MIN_PERF(~0L); 1156d4da12f8SPerry Yuan value |= AMD_CPPC_MIN_PERF(min_perf); 1157d4da12f8SPerry Yuan wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); 1158d4da12f8SPerry Yuan } else { 1159d4da12f8SPerry Yuan perf_ctrls.desired_perf = 0; 1160d4da12f8SPerry Yuan perf_ctrls.max_perf = min_perf; 1161d4da12f8SPerry Yuan perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); 1162d4da12f8SPerry Yuan cppc_set_perf(cpudata->cpu, &perf_ctrls); 1163d4da12f8SPerry Yuan } 1164d4da12f8SPerry Yuan mutex_unlock(&amd_pstate_limits_lock); 1165d4da12f8SPerry Yuan } 1166d4da12f8SPerry Yuan 1167d4da12f8SPerry Yuan static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) 1168d4da12f8SPerry Yuan { 1169d4da12f8SPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 1170d4da12f8SPerry Yuan 1171d4da12f8SPerry Yuan pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); 1172d4da12f8SPerry Yuan 1173d4da12f8SPerry Yuan if (cpudata->suspended) 1174d4da12f8SPerry Yuan return 0; 1175d4da12f8SPerry Yuan 1176d4da12f8SPerry Yuan if (cppc_state == AMD_PSTATE_ACTIVE) 1177d4da12f8SPerry Yuan amd_pstate_epp_offline(policy); 1178d4da12f8SPerry Yuan 1179d4da12f8SPerry Yuan return 0; 1180d4da12f8SPerry Yuan } 1181d4da12f8SPerry Yuan 1182ffa5096aSPerry Yuan static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) 1183ffa5096aSPerry Yuan { 1184ffa5096aSPerry Yuan cpufreq_verify_within_cpu_limits(policy); 1185ffa5096aSPerry Yuan pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); 1186ffa5096aSPerry Yuan return 0; 1187ffa5096aSPerry Yuan } 1188ffa5096aSPerry Yuan 118950ddd2f7SPerry Yuan static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) 119050ddd2f7SPerry Yuan { 119150ddd2f7SPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 119250ddd2f7SPerry Yuan int ret; 119350ddd2f7SPerry Yuan 119450ddd2f7SPerry Yuan /* avoid suspending when EPP is not enabled */ 119550ddd2f7SPerry Yuan if (cppc_state != AMD_PSTATE_ACTIVE) 119650ddd2f7SPerry Yuan return 0; 119750ddd2f7SPerry Yuan 119850ddd2f7SPerry Yuan /* set this flag to avoid setting core offline*/ 119950ddd2f7SPerry Yuan cpudata->suspended = true; 120050ddd2f7SPerry Yuan 120150ddd2f7SPerry Yuan /* disable CPPC in lowlevel firmware */ 120250ddd2f7SPerry Yuan ret = amd_pstate_enable(false); 120350ddd2f7SPerry Yuan if (ret) 120450ddd2f7SPerry Yuan pr_err("failed to suspend, return %d\n", ret); 120550ddd2f7SPerry Yuan 120650ddd2f7SPerry Yuan return 0; 120750ddd2f7SPerry Yuan } 120850ddd2f7SPerry Yuan 120950ddd2f7SPerry Yuan static int amd_pstate_epp_resume(struct cpufreq_policy *policy) 121050ddd2f7SPerry Yuan { 121150ddd2f7SPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 121250ddd2f7SPerry Yuan 121350ddd2f7SPerry Yuan if (cpudata->suspended) { 121450ddd2f7SPerry Yuan mutex_lock(&amd_pstate_limits_lock); 121550ddd2f7SPerry Yuan 121650ddd2f7SPerry Yuan /* enable amd pstate from suspend state*/ 121750ddd2f7SPerry Yuan amd_pstate_epp_reenable(cpudata); 121850ddd2f7SPerry Yuan 121950ddd2f7SPerry Yuan mutex_unlock(&amd_pstate_limits_lock); 122050ddd2f7SPerry Yuan 122150ddd2f7SPerry Yuan cpudata->suspended = false; 122250ddd2f7SPerry Yuan } 122350ddd2f7SPerry Yuan 122450ddd2f7SPerry Yuan return 0; 122550ddd2f7SPerry Yuan } 122650ddd2f7SPerry Yuan 1227ec437d71SHuang Rui static struct cpufreq_driver amd_pstate_driver = { 1228ec437d71SHuang Rui .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 1229ec437d71SHuang Rui .verify = amd_pstate_verify, 1230ec437d71SHuang Rui .target = amd_pstate_target, 1231ec437d71SHuang Rui .init = amd_pstate_cpu_init, 1232ec437d71SHuang Rui .exit = amd_pstate_cpu_exit, 1233b376471fSJinzhou Su .suspend = amd_pstate_cpu_suspend, 1234b376471fSJinzhou Su .resume = amd_pstate_cpu_resume, 123541271016SHuang Rui .set_boost = amd_pstate_set_boost, 1236ec437d71SHuang Rui .name = "amd-pstate", 1237ec4e3326SHuang Rui .attr = amd_pstate_attr, 1238ec437d71SHuang Rui }; 1239ec437d71SHuang Rui 1240ffa5096aSPerry Yuan static struct cpufreq_driver amd_pstate_epp_driver = { 1241ffa5096aSPerry Yuan .flags = CPUFREQ_CONST_LOOPS, 1242ffa5096aSPerry Yuan .verify = amd_pstate_epp_verify_policy, 1243ffa5096aSPerry Yuan .setpolicy = amd_pstate_epp_set_policy, 1244ffa5096aSPerry Yuan .init = amd_pstate_epp_cpu_init, 1245ffa5096aSPerry Yuan .exit = amd_pstate_epp_cpu_exit, 1246d4da12f8SPerry Yuan .offline = amd_pstate_epp_cpu_offline, 1247d4da12f8SPerry Yuan .online = amd_pstate_epp_cpu_online, 124850ddd2f7SPerry Yuan .suspend = amd_pstate_epp_suspend, 124950ddd2f7SPerry Yuan .resume = amd_pstate_epp_resume, 1250ffa5096aSPerry Yuan .name = "amd_pstate_epp", 1251ffa5096aSPerry Yuan .attr = amd_pstate_epp_attr, 1252ffa5096aSPerry Yuan }; 1253ffa5096aSPerry Yuan 1254ec437d71SHuang Rui static int __init amd_pstate_init(void) 1255ec437d71SHuang Rui { 1256ec437d71SHuang Rui int ret; 1257ec437d71SHuang Rui 1258ec437d71SHuang Rui if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 1259ec437d71SHuang Rui return -ENODEV; 1260202e683dSPerry Yuan /* 1261202e683dSPerry Yuan * by default the pstate driver is disabled to load 1262202e683dSPerry Yuan * enable the amd_pstate passive mode driver explicitly 126336c5014eSWyes Karny * with amd_pstate=passive or other modes in kernel command line 1264202e683dSPerry Yuan */ 126536c5014eSWyes Karny if (cppc_state == AMD_PSTATE_DISABLE) { 126636c5014eSWyes Karny pr_debug("driver load is disabled, boot with specific mode to enable this\n"); 1267202e683dSPerry Yuan return -ENODEV; 1268202e683dSPerry Yuan } 1269ec437d71SHuang Rui 1270ec437d71SHuang Rui if (!acpi_cpc_valid()) { 1271a2a9d185SPerry Yuan pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 1272ec437d71SHuang Rui return -ENODEV; 1273ec437d71SHuang Rui } 1274ec437d71SHuang Rui 1275ec437d71SHuang Rui /* don't keep reloading if cpufreq_driver exists */ 1276ec437d71SHuang Rui if (cpufreq_get_current_driver()) 1277ec437d71SHuang Rui return -EEXIST; 1278ec437d71SHuang Rui 1279ec437d71SHuang Rui /* capability check */ 1280e059c184SHuang Rui if (boot_cpu_has(X86_FEATURE_CPPC)) { 1281e059c184SHuang Rui pr_debug("AMD CPPC MSR based functionality is supported\n"); 1282ffa5096aSPerry Yuan if (cppc_state == AMD_PSTATE_PASSIVE) 1283ffa5096aSPerry Yuan current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; 1284202e683dSPerry Yuan } else { 1285202e683dSPerry Yuan pr_debug("AMD CPPC shared memory based functionality is supported\n"); 1286e059c184SHuang Rui static_call_update(amd_pstate_enable, cppc_enable); 1287e059c184SHuang Rui static_call_update(amd_pstate_init_perf, cppc_init_perf); 1288e059c184SHuang Rui static_call_update(amd_pstate_update_perf, cppc_update_perf); 1289ec437d71SHuang Rui } 1290ec437d71SHuang Rui 1291ec437d71SHuang Rui /* enable amd pstate feature */ 1292ec437d71SHuang Rui ret = amd_pstate_enable(true); 1293ec437d71SHuang Rui if (ret) { 1294ffa5096aSPerry Yuan pr_err("failed to enable with return %d\n", ret); 1295ec437d71SHuang Rui return ret; 1296ec437d71SHuang Rui } 1297ec437d71SHuang Rui 1298ffa5096aSPerry Yuan ret = cpufreq_register_driver(current_pstate_driver); 1299ec437d71SHuang Rui if (ret) 1300ffa5096aSPerry Yuan pr_err("failed to register with return %d\n", ret); 1301ec437d71SHuang Rui 1302abd61c08SPerry Yuan amd_pstate_kobj = kobject_create_and_add("amd_pstate", &cpu_subsys.dev_root->kobj); 1303abd61c08SPerry Yuan if (!amd_pstate_kobj) { 1304abd61c08SPerry Yuan ret = -EINVAL; 1305abd61c08SPerry Yuan pr_err("global sysfs registration failed.\n"); 1306abd61c08SPerry Yuan goto kobject_free; 1307abd61c08SPerry Yuan } 1308abd61c08SPerry Yuan 1309abd61c08SPerry Yuan ret = sysfs_create_group(amd_pstate_kobj, &amd_pstate_global_attr_group); 1310abd61c08SPerry Yuan if (ret) { 1311abd61c08SPerry Yuan pr_err("sysfs attribute export failed with error %d.\n", ret); 1312abd61c08SPerry Yuan goto global_attr_free; 1313abd61c08SPerry Yuan } 1314abd61c08SPerry Yuan 1315abd61c08SPerry Yuan return ret; 1316abd61c08SPerry Yuan 1317abd61c08SPerry Yuan global_attr_free: 1318abd61c08SPerry Yuan kobject_put(amd_pstate_kobj); 1319abd61c08SPerry Yuan kobject_free: 1320abd61c08SPerry Yuan cpufreq_unregister_driver(current_pstate_driver); 1321ec437d71SHuang Rui return ret; 1322ec437d71SHuang Rui } 1323456ca88dSPerry Yuan device_initcall(amd_pstate_init); 1324ec437d71SHuang Rui 1325202e683dSPerry Yuan static int __init amd_pstate_param(char *str) 1326202e683dSPerry Yuan { 132736c5014eSWyes Karny size_t size; 132836c5014eSWyes Karny int mode_idx; 132936c5014eSWyes Karny 1330202e683dSPerry Yuan if (!str) 1331202e683dSPerry Yuan return -EINVAL; 1332202e683dSPerry Yuan 133336c5014eSWyes Karny size = strlen(str); 133436c5014eSWyes Karny mode_idx = get_mode_idx_from_str(str, size); 133536c5014eSWyes Karny 133636c5014eSWyes Karny if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { 133736c5014eSWyes Karny cppc_state = mode_idx; 133836c5014eSWyes Karny if (cppc_state == AMD_PSTATE_DISABLE) 1339202e683dSPerry Yuan pr_info("driver is explicitly disabled\n"); 1340202e683dSPerry Yuan 1341ffa5096aSPerry Yuan if (cppc_state == AMD_PSTATE_ACTIVE) 1342ffa5096aSPerry Yuan current_pstate_driver = &amd_pstate_epp_driver; 1343ffa5096aSPerry Yuan 1344ffa5096aSPerry Yuan if (cppc_state == AMD_PSTATE_PASSIVE) 1345ffa5096aSPerry Yuan current_pstate_driver = &amd_pstate_driver; 1346ffa5096aSPerry Yuan 1347202e683dSPerry Yuan return 0; 1348202e683dSPerry Yuan } 134936c5014eSWyes Karny 135036c5014eSWyes Karny return -EINVAL; 135136c5014eSWyes Karny } 1352202e683dSPerry Yuan early_param("amd_pstate", amd_pstate_param); 1353202e683dSPerry Yuan 1354ec437d71SHuang Rui MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 1355ec437d71SHuang Rui MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 1356ec437d71SHuang Rui MODULE_LICENSE("GPL"); 1357