1ec437d71SHuang Rui // SPDX-License-Identifier: GPL-2.0-or-later 2ec437d71SHuang Rui /* 3ec437d71SHuang Rui * amd-pstate.c - AMD Processor P-state Frequency Driver 4ec437d71SHuang Rui * 5ec437d71SHuang Rui * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6ec437d71SHuang Rui * 7ec437d71SHuang Rui * Author: Huang Rui <ray.huang@amd.com> 8ec437d71SHuang Rui * 9ec437d71SHuang Rui * AMD P-State introduces a new CPU performance scaling design for AMD 10ec437d71SHuang Rui * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11ec437d71SHuang Rui * feature which works with the AMD SMU firmware providing a finer grained 12ec437d71SHuang Rui * frequency control range. It is to replace the legacy ACPI P-States control, 13ec437d71SHuang Rui * allows a flexible, low-latency interface for the Linux kernel to directly 14ec437d71SHuang Rui * communicate the performance hints to hardware. 15ec437d71SHuang Rui * 16ec437d71SHuang Rui * AMD P-State is supported on recent AMD Zen base CPU series include some of 17ec437d71SHuang Rui * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18ec437d71SHuang Rui * P-State supported system. And there are two types of hardware implementations 19ec437d71SHuang Rui * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20ec437d71SHuang Rui * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21ec437d71SHuang Rui */ 22ec437d71SHuang Rui 23ec437d71SHuang Rui #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24ec437d71SHuang Rui 25ec437d71SHuang Rui #include <linux/kernel.h> 26ec437d71SHuang Rui #include <linux/module.h> 27ec437d71SHuang Rui #include <linux/init.h> 28ec437d71SHuang Rui #include <linux/smp.h> 29ec437d71SHuang Rui #include <linux/sched.h> 30ec437d71SHuang Rui #include <linux/cpufreq.h> 31ec437d71SHuang Rui #include <linux/compiler.h> 32ec437d71SHuang Rui #include <linux/dmi.h> 33ec437d71SHuang Rui #include <linux/slab.h> 34ec437d71SHuang Rui #include <linux/acpi.h> 35ec437d71SHuang Rui #include <linux/io.h> 36ec437d71SHuang Rui #include <linux/delay.h> 37ec437d71SHuang Rui #include <linux/uaccess.h> 38ec437d71SHuang Rui #include <linux/static_call.h> 39ec437d71SHuang Rui 40ec437d71SHuang Rui #include <acpi/processor.h> 41ec437d71SHuang Rui #include <acpi/cppc_acpi.h> 42ec437d71SHuang Rui 43ec437d71SHuang Rui #include <asm/msr.h> 44ec437d71SHuang Rui #include <asm/processor.h> 45ec437d71SHuang Rui #include <asm/cpufeature.h> 46ec437d71SHuang Rui #include <asm/cpu_device_id.h> 4760e10f89SHuang Rui #include "amd-pstate-trace.h" 48ec437d71SHuang Rui 49ec437d71SHuang Rui #define AMD_PSTATE_TRANSITION_LATENCY 0x20000 50ec437d71SHuang Rui #define AMD_PSTATE_TRANSITION_DELAY 500 51ec437d71SHuang Rui 52e059c184SHuang Rui /* 53e059c184SHuang Rui * TODO: We need more time to fine tune processors with shared memory solution 54e059c184SHuang Rui * with community together. 55e059c184SHuang Rui * 56e059c184SHuang Rui * There are some performance drops on the CPU benchmarks which reports from 57e059c184SHuang Rui * Suse. We are co-working with them to fine tune the shared memory solution. So 58e059c184SHuang Rui * we disable it by default to go acpi-cpufreq on these processors and add a 59e059c184SHuang Rui * module parameter to be able to enable it manually for debugging. 60e059c184SHuang Rui */ 61e059c184SHuang Rui static bool shared_mem = false; 62e059c184SHuang Rui module_param(shared_mem, bool, 0444); 63e059c184SHuang Rui MODULE_PARM_DESC(shared_mem, 64e059c184SHuang Rui "enable amd-pstate on processors with shared memory solution (false = disabled (default), true = enabled)"); 65e059c184SHuang Rui 66ec437d71SHuang Rui static struct cpufreq_driver amd_pstate_driver; 67ec437d71SHuang Rui 68ec437d71SHuang Rui /** 69ec437d71SHuang Rui * struct amd_cpudata - private CPU data for AMD P-State 70ec437d71SHuang Rui * @cpu: CPU number 71ec437d71SHuang Rui * @cppc_req_cached: cached performance request hints 72ec437d71SHuang Rui * @highest_perf: the maximum performance an individual processor may reach, 73ec437d71SHuang Rui * assuming ideal conditions 74ec437d71SHuang Rui * @nominal_perf: the maximum sustained performance level of the processor, 75ec437d71SHuang Rui * assuming ideal operating conditions 76ec437d71SHuang Rui * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power 77ec437d71SHuang Rui * savings are achieved 78ec437d71SHuang Rui * @lowest_perf: the absolute lowest performance level of the processor 79ec437d71SHuang Rui * @max_freq: the frequency that mapped to highest_perf 80ec437d71SHuang Rui * @min_freq: the frequency that mapped to lowest_perf 81ec437d71SHuang Rui * @nominal_freq: the frequency that mapped to nominal_perf 82ec437d71SHuang Rui * @lowest_nonlinear_freq: the frequency that mapped to lowest_nonlinear_perf 83ec437d71SHuang Rui * 84ec437d71SHuang Rui * The amd_cpudata is key private data for each CPU thread in AMD P-State, and 85ec437d71SHuang Rui * represents all the attributes and goals that AMD P-State requests at runtime. 86ec437d71SHuang Rui */ 87ec437d71SHuang Rui struct amd_cpudata { 88ec437d71SHuang Rui int cpu; 89ec437d71SHuang Rui 9041271016SHuang Rui struct freq_qos_request req[2]; 91ec437d71SHuang Rui u64 cppc_req_cached; 92ec437d71SHuang Rui 93ec437d71SHuang Rui u32 highest_perf; 94ec437d71SHuang Rui u32 nominal_perf; 95ec437d71SHuang Rui u32 lowest_nonlinear_perf; 96ec437d71SHuang Rui u32 lowest_perf; 97ec437d71SHuang Rui 98ec437d71SHuang Rui u32 max_freq; 99ec437d71SHuang Rui u32 min_freq; 100ec437d71SHuang Rui u32 nominal_freq; 101ec437d71SHuang Rui u32 lowest_nonlinear_freq; 10241271016SHuang Rui 10341271016SHuang Rui bool boost_supported; 104ec437d71SHuang Rui }; 105ec437d71SHuang Rui 106e059c184SHuang Rui static inline int pstate_enable(bool enable) 107ec437d71SHuang Rui { 108ec437d71SHuang Rui return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); 109ec437d71SHuang Rui } 110ec437d71SHuang Rui 111e059c184SHuang Rui static int cppc_enable(bool enable) 112e059c184SHuang Rui { 113e059c184SHuang Rui int cpu, ret = 0; 114e059c184SHuang Rui 115e059c184SHuang Rui for_each_present_cpu(cpu) { 116e059c184SHuang Rui ret = cppc_set_enable(cpu, enable); 117e059c184SHuang Rui if (ret) 118e059c184SHuang Rui return ret; 119e059c184SHuang Rui } 120e059c184SHuang Rui 121e059c184SHuang Rui return ret; 122e059c184SHuang Rui } 123e059c184SHuang Rui 124e059c184SHuang Rui DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); 125e059c184SHuang Rui 126e059c184SHuang Rui static inline int amd_pstate_enable(bool enable) 127e059c184SHuang Rui { 128e059c184SHuang Rui return static_call(amd_pstate_enable)(enable); 129e059c184SHuang Rui } 130e059c184SHuang Rui 131e059c184SHuang Rui static int pstate_init_perf(struct amd_cpudata *cpudata) 132ec437d71SHuang Rui { 133ec437d71SHuang Rui u64 cap1; 134ec437d71SHuang Rui 135ec437d71SHuang Rui int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 136ec437d71SHuang Rui &cap1); 137ec437d71SHuang Rui if (ret) 138ec437d71SHuang Rui return ret; 139ec437d71SHuang Rui 140ec437d71SHuang Rui /* 141ec437d71SHuang Rui * TODO: Introduce AMD specific power feature. 142ec437d71SHuang Rui * 143ec437d71SHuang Rui * CPPC entry doesn't indicate the highest performance in some ASICs. 144ec437d71SHuang Rui */ 145ec437d71SHuang Rui WRITE_ONCE(cpudata->highest_perf, amd_get_highest_perf()); 146ec437d71SHuang Rui 147ec437d71SHuang Rui WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); 148ec437d71SHuang Rui WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); 149ec437d71SHuang Rui WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); 150ec437d71SHuang Rui 151ec437d71SHuang Rui return 0; 152ec437d71SHuang Rui } 153ec437d71SHuang Rui 154e059c184SHuang Rui static int cppc_init_perf(struct amd_cpudata *cpudata) 155e059c184SHuang Rui { 156e059c184SHuang Rui struct cppc_perf_caps cppc_perf; 157e059c184SHuang Rui 158e059c184SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 159e059c184SHuang Rui if (ret) 160e059c184SHuang Rui return ret; 161e059c184SHuang Rui 162e059c184SHuang Rui WRITE_ONCE(cpudata->highest_perf, amd_get_highest_perf()); 163e059c184SHuang Rui 164e059c184SHuang Rui WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); 165e059c184SHuang Rui WRITE_ONCE(cpudata->lowest_nonlinear_perf, 166e059c184SHuang Rui cppc_perf.lowest_nonlinear_perf); 167e059c184SHuang Rui WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); 168e059c184SHuang Rui 169e059c184SHuang Rui return 0; 170e059c184SHuang Rui } 171e059c184SHuang Rui 172e059c184SHuang Rui DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); 173e059c184SHuang Rui 174e059c184SHuang Rui static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 175e059c184SHuang Rui { 176e059c184SHuang Rui return static_call(amd_pstate_init_perf)(cpudata); 177e059c184SHuang Rui } 178e059c184SHuang Rui 179e059c184SHuang Rui static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, 180ec437d71SHuang Rui u32 des_perf, u32 max_perf, bool fast_switch) 181ec437d71SHuang Rui { 182ec437d71SHuang Rui if (fast_switch) 183ec437d71SHuang Rui wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); 184ec437d71SHuang Rui else 185ec437d71SHuang Rui wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 186ec437d71SHuang Rui READ_ONCE(cpudata->cppc_req_cached)); 187ec437d71SHuang Rui } 188ec437d71SHuang Rui 189e059c184SHuang Rui static void cppc_update_perf(struct amd_cpudata *cpudata, 190e059c184SHuang Rui u32 min_perf, u32 des_perf, 191e059c184SHuang Rui u32 max_perf, bool fast_switch) 192e059c184SHuang Rui { 193e059c184SHuang Rui struct cppc_perf_ctrls perf_ctrls; 194e059c184SHuang Rui 195e059c184SHuang Rui perf_ctrls.max_perf = max_perf; 196e059c184SHuang Rui perf_ctrls.min_perf = min_perf; 197e059c184SHuang Rui perf_ctrls.desired_perf = des_perf; 198e059c184SHuang Rui 199e059c184SHuang Rui cppc_set_perf(cpudata->cpu, &perf_ctrls); 200e059c184SHuang Rui } 201e059c184SHuang Rui 202e059c184SHuang Rui DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); 203e059c184SHuang Rui 204e059c184SHuang Rui static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, 205e059c184SHuang Rui u32 min_perf, u32 des_perf, 206e059c184SHuang Rui u32 max_perf, bool fast_switch) 207e059c184SHuang Rui { 208e059c184SHuang Rui static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, 209e059c184SHuang Rui max_perf, fast_switch); 210e059c184SHuang Rui } 211e059c184SHuang Rui 212ec437d71SHuang Rui static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, 213ec437d71SHuang Rui u32 des_perf, u32 max_perf, bool fast_switch) 214ec437d71SHuang Rui { 215ec437d71SHuang Rui u64 prev = READ_ONCE(cpudata->cppc_req_cached); 216ec437d71SHuang Rui u64 value = prev; 217ec437d71SHuang Rui 218ec437d71SHuang Rui value &= ~AMD_CPPC_MIN_PERF(~0L); 219ec437d71SHuang Rui value |= AMD_CPPC_MIN_PERF(min_perf); 220ec437d71SHuang Rui 221ec437d71SHuang Rui value &= ~AMD_CPPC_DES_PERF(~0L); 222ec437d71SHuang Rui value |= AMD_CPPC_DES_PERF(des_perf); 223ec437d71SHuang Rui 224ec437d71SHuang Rui value &= ~AMD_CPPC_MAX_PERF(~0L); 225ec437d71SHuang Rui value |= AMD_CPPC_MAX_PERF(max_perf); 226ec437d71SHuang Rui 22760e10f89SHuang Rui trace_amd_pstate_perf(min_perf, des_perf, max_perf, 22860e10f89SHuang Rui cpudata->cpu, (value != prev), fast_switch); 22960e10f89SHuang Rui 230ec437d71SHuang Rui if (value == prev) 231ec437d71SHuang Rui return; 232ec437d71SHuang Rui 233ec437d71SHuang Rui WRITE_ONCE(cpudata->cppc_req_cached, value); 234ec437d71SHuang Rui 235ec437d71SHuang Rui amd_pstate_update_perf(cpudata, min_perf, des_perf, 236ec437d71SHuang Rui max_perf, fast_switch); 237ec437d71SHuang Rui } 238ec437d71SHuang Rui 239ec437d71SHuang Rui static int amd_pstate_verify(struct cpufreq_policy_data *policy) 240ec437d71SHuang Rui { 241ec437d71SHuang Rui cpufreq_verify_within_cpu_limits(policy); 242ec437d71SHuang Rui 243ec437d71SHuang Rui return 0; 244ec437d71SHuang Rui } 245ec437d71SHuang Rui 246ec437d71SHuang Rui static int amd_pstate_target(struct cpufreq_policy *policy, 247ec437d71SHuang Rui unsigned int target_freq, 248ec437d71SHuang Rui unsigned int relation) 249ec437d71SHuang Rui { 250ec437d71SHuang Rui struct cpufreq_freqs freqs; 251ec437d71SHuang Rui struct amd_cpudata *cpudata = policy->driver_data; 252ec437d71SHuang Rui unsigned long max_perf, min_perf, des_perf, cap_perf; 253ec437d71SHuang Rui 254ec437d71SHuang Rui if (!cpudata->max_freq) 255ec437d71SHuang Rui return -ENODEV; 256ec437d71SHuang Rui 257ec437d71SHuang Rui cap_perf = READ_ONCE(cpudata->highest_perf); 258ec437d71SHuang Rui min_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); 259ec437d71SHuang Rui max_perf = cap_perf; 260ec437d71SHuang Rui 261ec437d71SHuang Rui freqs.old = policy->cur; 262ec437d71SHuang Rui freqs.new = target_freq; 263ec437d71SHuang Rui 264ec437d71SHuang Rui des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, 265ec437d71SHuang Rui cpudata->max_freq); 266ec437d71SHuang Rui 267ec437d71SHuang Rui cpufreq_freq_transition_begin(policy, &freqs); 268ec437d71SHuang Rui amd_pstate_update(cpudata, min_perf, des_perf, 269ec437d71SHuang Rui max_perf, false); 270ec437d71SHuang Rui cpufreq_freq_transition_end(policy, &freqs, false); 271ec437d71SHuang Rui 272ec437d71SHuang Rui return 0; 273ec437d71SHuang Rui } 274ec437d71SHuang Rui 2751d215f03SHuang Rui static void amd_pstate_adjust_perf(unsigned int cpu, 2761d215f03SHuang Rui unsigned long _min_perf, 2771d215f03SHuang Rui unsigned long target_perf, 2781d215f03SHuang Rui unsigned long capacity) 2791d215f03SHuang Rui { 2801d215f03SHuang Rui unsigned long max_perf, min_perf, des_perf, 2811d215f03SHuang Rui cap_perf, lowest_nonlinear_perf; 2821d215f03SHuang Rui struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 2831d215f03SHuang Rui struct amd_cpudata *cpudata = policy->driver_data; 2841d215f03SHuang Rui 2851d215f03SHuang Rui cap_perf = READ_ONCE(cpudata->highest_perf); 2861d215f03SHuang Rui lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); 2871d215f03SHuang Rui 2881d215f03SHuang Rui des_perf = cap_perf; 2891d215f03SHuang Rui if (target_perf < capacity) 2901d215f03SHuang Rui des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 2911d215f03SHuang Rui 2921d215f03SHuang Rui min_perf = READ_ONCE(cpudata->highest_perf); 2931d215f03SHuang Rui if (_min_perf < capacity) 2941d215f03SHuang Rui min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 2951d215f03SHuang Rui 2961d215f03SHuang Rui if (min_perf < lowest_nonlinear_perf) 2971d215f03SHuang Rui min_perf = lowest_nonlinear_perf; 2981d215f03SHuang Rui 2991d215f03SHuang Rui max_perf = cap_perf; 3001d215f03SHuang Rui if (max_perf < min_perf) 3011d215f03SHuang Rui max_perf = min_perf; 3021d215f03SHuang Rui 3031d215f03SHuang Rui des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 3041d215f03SHuang Rui 3051d215f03SHuang Rui amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); 3061d215f03SHuang Rui } 3071d215f03SHuang Rui 308ec437d71SHuang Rui static int amd_get_min_freq(struct amd_cpudata *cpudata) 309ec437d71SHuang Rui { 310ec437d71SHuang Rui struct cppc_perf_caps cppc_perf; 311ec437d71SHuang Rui 312ec437d71SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 313ec437d71SHuang Rui if (ret) 314ec437d71SHuang Rui return ret; 315ec437d71SHuang Rui 316ec437d71SHuang Rui /* Switch to khz */ 317ec437d71SHuang Rui return cppc_perf.lowest_freq * 1000; 318ec437d71SHuang Rui } 319ec437d71SHuang Rui 320ec437d71SHuang Rui static int amd_get_max_freq(struct amd_cpudata *cpudata) 321ec437d71SHuang Rui { 322ec437d71SHuang Rui struct cppc_perf_caps cppc_perf; 323ec437d71SHuang Rui u32 max_perf, max_freq, nominal_freq, nominal_perf; 324ec437d71SHuang Rui u64 boost_ratio; 325ec437d71SHuang Rui 326ec437d71SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 327ec437d71SHuang Rui if (ret) 328ec437d71SHuang Rui return ret; 329ec437d71SHuang Rui 330ec437d71SHuang Rui nominal_freq = cppc_perf.nominal_freq; 331ec437d71SHuang Rui nominal_perf = READ_ONCE(cpudata->nominal_perf); 332ec437d71SHuang Rui max_perf = READ_ONCE(cpudata->highest_perf); 333ec437d71SHuang Rui 334ec437d71SHuang Rui boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT, 335ec437d71SHuang Rui nominal_perf); 336ec437d71SHuang Rui 337ec437d71SHuang Rui max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT; 338ec437d71SHuang Rui 339ec437d71SHuang Rui /* Switch to khz */ 340ec437d71SHuang Rui return max_freq * 1000; 341ec437d71SHuang Rui } 342ec437d71SHuang Rui 343ec437d71SHuang Rui static int amd_get_nominal_freq(struct amd_cpudata *cpudata) 344ec437d71SHuang Rui { 345ec437d71SHuang Rui struct cppc_perf_caps cppc_perf; 346ec437d71SHuang Rui 347ec437d71SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 348ec437d71SHuang Rui if (ret) 349ec437d71SHuang Rui return ret; 350ec437d71SHuang Rui 351ec437d71SHuang Rui /* Switch to khz */ 352ec437d71SHuang Rui return cppc_perf.nominal_freq * 1000; 353ec437d71SHuang Rui } 354ec437d71SHuang Rui 355ec437d71SHuang Rui static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) 356ec437d71SHuang Rui { 357ec437d71SHuang Rui struct cppc_perf_caps cppc_perf; 358ec437d71SHuang Rui u32 lowest_nonlinear_freq, lowest_nonlinear_perf, 359ec437d71SHuang Rui nominal_freq, nominal_perf; 360ec437d71SHuang Rui u64 lowest_nonlinear_ratio; 361ec437d71SHuang Rui 362ec437d71SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 363ec437d71SHuang Rui if (ret) 364ec437d71SHuang Rui return ret; 365ec437d71SHuang Rui 366ec437d71SHuang Rui nominal_freq = cppc_perf.nominal_freq; 367ec437d71SHuang Rui nominal_perf = READ_ONCE(cpudata->nominal_perf); 368ec437d71SHuang Rui 369ec437d71SHuang Rui lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 370ec437d71SHuang Rui 371ec437d71SHuang Rui lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, 372ec437d71SHuang Rui nominal_perf); 373ec437d71SHuang Rui 374ec437d71SHuang Rui lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT; 375ec437d71SHuang Rui 376ec437d71SHuang Rui /* Switch to khz */ 377ec437d71SHuang Rui return lowest_nonlinear_freq * 1000; 378ec437d71SHuang Rui } 379ec437d71SHuang Rui 38041271016SHuang Rui static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 38141271016SHuang Rui { 38241271016SHuang Rui struct amd_cpudata *cpudata = policy->driver_data; 38341271016SHuang Rui int ret; 38441271016SHuang Rui 38541271016SHuang Rui if (!cpudata->boost_supported) { 38641271016SHuang Rui pr_err("Boost mode is not supported by this processor or SBIOS\n"); 38741271016SHuang Rui return -EINVAL; 38841271016SHuang Rui } 38941271016SHuang Rui 39041271016SHuang Rui if (state) 39141271016SHuang Rui policy->cpuinfo.max_freq = cpudata->max_freq; 39241271016SHuang Rui else 39341271016SHuang Rui policy->cpuinfo.max_freq = cpudata->nominal_freq; 39441271016SHuang Rui 39541271016SHuang Rui policy->max = policy->cpuinfo.max_freq; 39641271016SHuang Rui 39741271016SHuang Rui ret = freq_qos_update_request(&cpudata->req[1], 39841271016SHuang Rui policy->cpuinfo.max_freq); 39941271016SHuang Rui if (ret < 0) 40041271016SHuang Rui return ret; 40141271016SHuang Rui 40241271016SHuang Rui return 0; 40341271016SHuang Rui } 40441271016SHuang Rui 40541271016SHuang Rui static void amd_pstate_boost_init(struct amd_cpudata *cpudata) 40641271016SHuang Rui { 40741271016SHuang Rui u32 highest_perf, nominal_perf; 40841271016SHuang Rui 40941271016SHuang Rui highest_perf = READ_ONCE(cpudata->highest_perf); 41041271016SHuang Rui nominal_perf = READ_ONCE(cpudata->nominal_perf); 41141271016SHuang Rui 41241271016SHuang Rui if (highest_perf <= nominal_perf) 41341271016SHuang Rui return; 41441271016SHuang Rui 41541271016SHuang Rui cpudata->boost_supported = true; 41641271016SHuang Rui amd_pstate_driver.boost_enabled = true; 41741271016SHuang Rui } 41841271016SHuang Rui 419ec437d71SHuang Rui static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 420ec437d71SHuang Rui { 421ec437d71SHuang Rui int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 422ec437d71SHuang Rui struct device *dev; 423ec437d71SHuang Rui struct amd_cpudata *cpudata; 424ec437d71SHuang Rui 425ec437d71SHuang Rui dev = get_cpu_device(policy->cpu); 426ec437d71SHuang Rui if (!dev) 427ec437d71SHuang Rui return -ENODEV; 428ec437d71SHuang Rui 429ec437d71SHuang Rui cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 430ec437d71SHuang Rui if (!cpudata) 431ec437d71SHuang Rui return -ENOMEM; 432ec437d71SHuang Rui 433ec437d71SHuang Rui cpudata->cpu = policy->cpu; 434ec437d71SHuang Rui 435ec437d71SHuang Rui ret = amd_pstate_init_perf(cpudata); 436ec437d71SHuang Rui if (ret) 43741271016SHuang Rui goto free_cpudata1; 438ec437d71SHuang Rui 439ec437d71SHuang Rui min_freq = amd_get_min_freq(cpudata); 440ec437d71SHuang Rui max_freq = amd_get_max_freq(cpudata); 441ec437d71SHuang Rui nominal_freq = amd_get_nominal_freq(cpudata); 442ec437d71SHuang Rui lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 443ec437d71SHuang Rui 444ec437d71SHuang Rui if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 445ec437d71SHuang Rui dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 446ec437d71SHuang Rui min_freq, max_freq); 447ec437d71SHuang Rui ret = -EINVAL; 44841271016SHuang Rui goto free_cpudata1; 449ec437d71SHuang Rui } 450ec437d71SHuang Rui 451ec437d71SHuang Rui policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; 452ec437d71SHuang Rui policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; 453ec437d71SHuang Rui 454ec437d71SHuang Rui policy->min = min_freq; 455ec437d71SHuang Rui policy->max = max_freq; 456ec437d71SHuang Rui 457ec437d71SHuang Rui policy->cpuinfo.min_freq = min_freq; 458ec437d71SHuang Rui policy->cpuinfo.max_freq = max_freq; 459ec437d71SHuang Rui 460ec437d71SHuang Rui /* It will be updated by governor */ 461ec437d71SHuang Rui policy->cur = policy->cpuinfo.min_freq; 462ec437d71SHuang Rui 463e059c184SHuang Rui if (boot_cpu_has(X86_FEATURE_CPPC)) 4641d215f03SHuang Rui policy->fast_switch_possible = true; 4651d215f03SHuang Rui 46641271016SHuang Rui ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 46741271016SHuang Rui FREQ_QOS_MIN, policy->cpuinfo.min_freq); 46841271016SHuang Rui if (ret < 0) { 46941271016SHuang Rui dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 47041271016SHuang Rui goto free_cpudata1; 47141271016SHuang Rui } 47241271016SHuang Rui 47341271016SHuang Rui ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 47441271016SHuang Rui FREQ_QOS_MAX, policy->cpuinfo.max_freq); 47541271016SHuang Rui if (ret < 0) { 47641271016SHuang Rui dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 47741271016SHuang Rui goto free_cpudata2; 47841271016SHuang Rui } 47941271016SHuang Rui 480ec437d71SHuang Rui /* Initial processor data capability frequencies */ 481ec437d71SHuang Rui cpudata->max_freq = max_freq; 482ec437d71SHuang Rui cpudata->min_freq = min_freq; 483ec437d71SHuang Rui cpudata->nominal_freq = nominal_freq; 484ec437d71SHuang Rui cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 485ec437d71SHuang Rui 486ec437d71SHuang Rui policy->driver_data = cpudata; 487ec437d71SHuang Rui 48841271016SHuang Rui amd_pstate_boost_init(cpudata); 48941271016SHuang Rui 490ec437d71SHuang Rui return 0; 491ec437d71SHuang Rui 49241271016SHuang Rui free_cpudata2: 49341271016SHuang Rui freq_qos_remove_request(&cpudata->req[0]); 49441271016SHuang Rui free_cpudata1: 495ec437d71SHuang Rui kfree(cpudata); 496ec437d71SHuang Rui return ret; 497ec437d71SHuang Rui } 498ec437d71SHuang Rui 499ec437d71SHuang Rui static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) 500ec437d71SHuang Rui { 501ec437d71SHuang Rui struct amd_cpudata *cpudata; 502ec437d71SHuang Rui 503ec437d71SHuang Rui cpudata = policy->driver_data; 504ec437d71SHuang Rui 50541271016SHuang Rui freq_qos_remove_request(&cpudata->req[1]); 50641271016SHuang Rui freq_qos_remove_request(&cpudata->req[0]); 507ec437d71SHuang Rui kfree(cpudata); 508ec437d71SHuang Rui 509ec437d71SHuang Rui return 0; 510ec437d71SHuang Rui } 511ec437d71SHuang Rui 512ec4e3326SHuang Rui /* Sysfs attributes */ 513ec4e3326SHuang Rui 514ec4e3326SHuang Rui /* 515ec4e3326SHuang Rui * This frequency is to indicate the maximum hardware frequency. 516ec4e3326SHuang Rui * If boost is not active but supported, the frequency will be larger than the 517ec4e3326SHuang Rui * one in cpuinfo. 518ec4e3326SHuang Rui */ 519ec4e3326SHuang Rui static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 520ec4e3326SHuang Rui char *buf) 521ec4e3326SHuang Rui { 522ec4e3326SHuang Rui int max_freq; 523ec4e3326SHuang Rui struct amd_cpudata *cpudata; 524ec4e3326SHuang Rui 525ec4e3326SHuang Rui cpudata = policy->driver_data; 526ec4e3326SHuang Rui 527ec4e3326SHuang Rui max_freq = amd_get_max_freq(cpudata); 528ec4e3326SHuang Rui if (max_freq < 0) 529ec4e3326SHuang Rui return max_freq; 530ec4e3326SHuang Rui 531ec4e3326SHuang Rui return sprintf(&buf[0], "%u\n", max_freq); 532ec4e3326SHuang Rui } 533ec4e3326SHuang Rui 534ec4e3326SHuang Rui static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 535ec4e3326SHuang Rui char *buf) 536ec4e3326SHuang Rui { 537ec4e3326SHuang Rui int freq; 538ec4e3326SHuang Rui struct amd_cpudata *cpudata; 539ec4e3326SHuang Rui 540ec4e3326SHuang Rui cpudata = policy->driver_data; 541ec4e3326SHuang Rui 542ec4e3326SHuang Rui freq = amd_get_lowest_nonlinear_freq(cpudata); 543ec4e3326SHuang Rui if (freq < 0) 544ec4e3326SHuang Rui return freq; 545ec4e3326SHuang Rui 546ec4e3326SHuang Rui return sprintf(&buf[0], "%u\n", freq); 547ec4e3326SHuang Rui } 548ec4e3326SHuang Rui 549*3ad7fde1SHuang Rui /* 550*3ad7fde1SHuang Rui * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 551*3ad7fde1SHuang Rui * need to expose it to sysfs. 552*3ad7fde1SHuang Rui */ 553*3ad7fde1SHuang Rui static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 554*3ad7fde1SHuang Rui char *buf) 555*3ad7fde1SHuang Rui { 556*3ad7fde1SHuang Rui u32 perf; 557*3ad7fde1SHuang Rui struct amd_cpudata *cpudata = policy->driver_data; 558*3ad7fde1SHuang Rui 559*3ad7fde1SHuang Rui perf = READ_ONCE(cpudata->highest_perf); 560*3ad7fde1SHuang Rui 561*3ad7fde1SHuang Rui return sprintf(&buf[0], "%u\n", perf); 562*3ad7fde1SHuang Rui } 563*3ad7fde1SHuang Rui 564ec4e3326SHuang Rui cpufreq_freq_attr_ro(amd_pstate_max_freq); 565ec4e3326SHuang Rui cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 566ec4e3326SHuang Rui 567*3ad7fde1SHuang Rui cpufreq_freq_attr_ro(amd_pstate_highest_perf); 568*3ad7fde1SHuang Rui 569ec4e3326SHuang Rui static struct freq_attr *amd_pstate_attr[] = { 570ec4e3326SHuang Rui &amd_pstate_max_freq, 571ec4e3326SHuang Rui &amd_pstate_lowest_nonlinear_freq, 572*3ad7fde1SHuang Rui &amd_pstate_highest_perf, 573ec4e3326SHuang Rui NULL, 574ec4e3326SHuang Rui }; 575ec4e3326SHuang Rui 576ec437d71SHuang Rui static struct cpufreq_driver amd_pstate_driver = { 577ec437d71SHuang Rui .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 578ec437d71SHuang Rui .verify = amd_pstate_verify, 579ec437d71SHuang Rui .target = amd_pstate_target, 580ec437d71SHuang Rui .init = amd_pstate_cpu_init, 581ec437d71SHuang Rui .exit = amd_pstate_cpu_exit, 58241271016SHuang Rui .set_boost = amd_pstate_set_boost, 583ec437d71SHuang Rui .name = "amd-pstate", 584ec4e3326SHuang Rui .attr = amd_pstate_attr, 585ec437d71SHuang Rui }; 586ec437d71SHuang Rui 587ec437d71SHuang Rui static int __init amd_pstate_init(void) 588ec437d71SHuang Rui { 589ec437d71SHuang Rui int ret; 590ec437d71SHuang Rui 591ec437d71SHuang Rui if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 592ec437d71SHuang Rui return -ENODEV; 593ec437d71SHuang Rui 594ec437d71SHuang Rui if (!acpi_cpc_valid()) { 595ec437d71SHuang Rui pr_debug("the _CPC object is not present in SBIOS\n"); 596ec437d71SHuang Rui return -ENODEV; 597ec437d71SHuang Rui } 598ec437d71SHuang Rui 599ec437d71SHuang Rui /* don't keep reloading if cpufreq_driver exists */ 600ec437d71SHuang Rui if (cpufreq_get_current_driver()) 601ec437d71SHuang Rui return -EEXIST; 602ec437d71SHuang Rui 603ec437d71SHuang Rui /* capability check */ 604e059c184SHuang Rui if (boot_cpu_has(X86_FEATURE_CPPC)) { 605e059c184SHuang Rui pr_debug("AMD CPPC MSR based functionality is supported\n"); 606e059c184SHuang Rui amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf; 607e059c184SHuang Rui } else if (shared_mem) { 608e059c184SHuang Rui static_call_update(amd_pstate_enable, cppc_enable); 609e059c184SHuang Rui static_call_update(amd_pstate_init_perf, cppc_init_perf); 610e059c184SHuang Rui static_call_update(amd_pstate_update_perf, cppc_update_perf); 611e059c184SHuang Rui } else { 612e059c184SHuang Rui pr_info("This processor supports shared memory solution, you can enable it with amd_pstate.shared_mem=1\n"); 613ec437d71SHuang Rui return -ENODEV; 614ec437d71SHuang Rui } 615ec437d71SHuang Rui 616ec437d71SHuang Rui /* enable amd pstate feature */ 617ec437d71SHuang Rui ret = amd_pstate_enable(true); 618ec437d71SHuang Rui if (ret) { 619ec437d71SHuang Rui pr_err("failed to enable amd-pstate with return %d\n", ret); 620ec437d71SHuang Rui return ret; 621ec437d71SHuang Rui } 622ec437d71SHuang Rui 623ec437d71SHuang Rui ret = cpufreq_register_driver(&amd_pstate_driver); 624ec437d71SHuang Rui if (ret) 625ec437d71SHuang Rui pr_err("failed to register amd_pstate_driver with return %d\n", 626ec437d71SHuang Rui ret); 627ec437d71SHuang Rui 628ec437d71SHuang Rui return ret; 629ec437d71SHuang Rui } 630ec437d71SHuang Rui 631ec437d71SHuang Rui static void __exit amd_pstate_exit(void) 632ec437d71SHuang Rui { 633ec437d71SHuang Rui cpufreq_unregister_driver(&amd_pstate_driver); 634ec437d71SHuang Rui 635ec437d71SHuang Rui amd_pstate_enable(false); 636ec437d71SHuang Rui } 637ec437d71SHuang Rui 638ec437d71SHuang Rui module_init(amd_pstate_init); 639ec437d71SHuang Rui module_exit(amd_pstate_exit); 640ec437d71SHuang Rui 641ec437d71SHuang Rui MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 642ec437d71SHuang Rui MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 643ec437d71SHuang Rui MODULE_LICENSE("GPL"); 644