1ec437d71SHuang Rui // SPDX-License-Identifier: GPL-2.0-or-later 2ec437d71SHuang Rui /* 3ec437d71SHuang Rui * amd-pstate.c - AMD Processor P-state Frequency Driver 4ec437d71SHuang Rui * 5ec437d71SHuang Rui * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. 6ec437d71SHuang Rui * 7ec437d71SHuang Rui * Author: Huang Rui <ray.huang@amd.com> 8ec437d71SHuang Rui * 9ec437d71SHuang Rui * AMD P-State introduces a new CPU performance scaling design for AMD 10ec437d71SHuang Rui * processors using the ACPI Collaborative Performance and Power Control (CPPC) 11ec437d71SHuang Rui * feature which works with the AMD SMU firmware providing a finer grained 12ec437d71SHuang Rui * frequency control range. It is to replace the legacy ACPI P-States control, 13ec437d71SHuang Rui * allows a flexible, low-latency interface for the Linux kernel to directly 14ec437d71SHuang Rui * communicate the performance hints to hardware. 15ec437d71SHuang Rui * 16ec437d71SHuang Rui * AMD P-State is supported on recent AMD Zen base CPU series include some of 17ec437d71SHuang Rui * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD 18ec437d71SHuang Rui * P-State supported system. And there are two types of hardware implementations 19ec437d71SHuang Rui * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. 20ec437d71SHuang Rui * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. 21ec437d71SHuang Rui */ 22ec437d71SHuang Rui 23ec437d71SHuang Rui #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 24ec437d71SHuang Rui 25ec437d71SHuang Rui #include <linux/kernel.h> 26ec437d71SHuang Rui #include <linux/module.h> 27ec437d71SHuang Rui #include <linux/init.h> 28ec437d71SHuang Rui #include <linux/smp.h> 29ec437d71SHuang Rui #include <linux/sched.h> 30ec437d71SHuang Rui #include <linux/cpufreq.h> 31ec437d71SHuang Rui #include <linux/compiler.h> 32ec437d71SHuang Rui #include <linux/dmi.h> 33ec437d71SHuang Rui #include <linux/slab.h> 34ec437d71SHuang Rui #include <linux/acpi.h> 35ec437d71SHuang Rui #include <linux/io.h> 36ec437d71SHuang Rui #include <linux/delay.h> 37ec437d71SHuang Rui #include <linux/uaccess.h> 38ec437d71SHuang Rui #include <linux/static_call.h> 39f1375ec1SMeng Li #include <linux/amd-pstate.h> 40ec437d71SHuang Rui 41ec437d71SHuang Rui #include <acpi/processor.h> 42ec437d71SHuang Rui #include <acpi/cppc_acpi.h> 43ec437d71SHuang Rui 44ec437d71SHuang Rui #include <asm/msr.h> 45ec437d71SHuang Rui #include <asm/processor.h> 46ec437d71SHuang Rui #include <asm/cpufeature.h> 47ec437d71SHuang Rui #include <asm/cpu_device_id.h> 4860e10f89SHuang Rui #include "amd-pstate-trace.h" 49ec437d71SHuang Rui 50ca08e46dSPerry Yuan #define AMD_PSTATE_TRANSITION_LATENCY 20000 51ca08e46dSPerry Yuan #define AMD_PSTATE_TRANSITION_DELAY 1000 52ec437d71SHuang Rui 53e059c184SHuang Rui /* 54e059c184SHuang Rui * TODO: We need more time to fine tune processors with shared memory solution 55e059c184SHuang Rui * with community together. 56e059c184SHuang Rui * 57e059c184SHuang Rui * There are some performance drops on the CPU benchmarks which reports from 58e059c184SHuang Rui * Suse. We are co-working with them to fine tune the shared memory solution. So 59e059c184SHuang Rui * we disable it by default to go acpi-cpufreq on these processors and add a 60e059c184SHuang Rui * module parameter to be able to enable it manually for debugging. 61e059c184SHuang Rui */ 62ec437d71SHuang Rui static struct cpufreq_driver amd_pstate_driver; 63*202e683dSPerry Yuan static int cppc_load __initdata; 64ec437d71SHuang Rui 65e059c184SHuang Rui static inline int pstate_enable(bool enable) 66ec437d71SHuang Rui { 67ec437d71SHuang Rui return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); 68ec437d71SHuang Rui } 69ec437d71SHuang Rui 70e059c184SHuang Rui static int cppc_enable(bool enable) 71e059c184SHuang Rui { 72e059c184SHuang Rui int cpu, ret = 0; 73e059c184SHuang Rui 74e059c184SHuang Rui for_each_present_cpu(cpu) { 75e059c184SHuang Rui ret = cppc_set_enable(cpu, enable); 76e059c184SHuang Rui if (ret) 77e059c184SHuang Rui return ret; 78e059c184SHuang Rui } 79e059c184SHuang Rui 80e059c184SHuang Rui return ret; 81e059c184SHuang Rui } 82e059c184SHuang Rui 83e059c184SHuang Rui DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); 84e059c184SHuang Rui 85e059c184SHuang Rui static inline int amd_pstate_enable(bool enable) 86e059c184SHuang Rui { 87e059c184SHuang Rui return static_call(amd_pstate_enable)(enable); 88e059c184SHuang Rui } 89e059c184SHuang Rui 90e059c184SHuang Rui static int pstate_init_perf(struct amd_cpudata *cpudata) 91ec437d71SHuang Rui { 92ec437d71SHuang Rui u64 cap1; 93bedadcfbSPerry Yuan u32 highest_perf; 94ec437d71SHuang Rui 95ec437d71SHuang Rui int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, 96ec437d71SHuang Rui &cap1); 97ec437d71SHuang Rui if (ret) 98ec437d71SHuang Rui return ret; 99ec437d71SHuang Rui 100ec437d71SHuang Rui /* 101ec437d71SHuang Rui * TODO: Introduce AMD specific power feature. 102ec437d71SHuang Rui * 103ec437d71SHuang Rui * CPPC entry doesn't indicate the highest performance in some ASICs. 104ec437d71SHuang Rui */ 105bedadcfbSPerry Yuan highest_perf = amd_get_highest_perf(); 106bedadcfbSPerry Yuan if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) 107bedadcfbSPerry Yuan highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); 108bedadcfbSPerry Yuan 109bedadcfbSPerry Yuan WRITE_ONCE(cpudata->highest_perf, highest_perf); 110ec437d71SHuang Rui 111ec437d71SHuang Rui WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); 112ec437d71SHuang Rui WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); 113ec437d71SHuang Rui WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); 114ec437d71SHuang Rui 115ec437d71SHuang Rui return 0; 116ec437d71SHuang Rui } 117ec437d71SHuang Rui 118e059c184SHuang Rui static int cppc_init_perf(struct amd_cpudata *cpudata) 119e059c184SHuang Rui { 120e059c184SHuang Rui struct cppc_perf_caps cppc_perf; 121bedadcfbSPerry Yuan u32 highest_perf; 122e059c184SHuang Rui 123e059c184SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 124e059c184SHuang Rui if (ret) 125e059c184SHuang Rui return ret; 126e059c184SHuang Rui 127bedadcfbSPerry Yuan highest_perf = amd_get_highest_perf(); 128bedadcfbSPerry Yuan if (highest_perf > cppc_perf.highest_perf) 129bedadcfbSPerry Yuan highest_perf = cppc_perf.highest_perf; 130bedadcfbSPerry Yuan 131bedadcfbSPerry Yuan WRITE_ONCE(cpudata->highest_perf, highest_perf); 132e059c184SHuang Rui 133e059c184SHuang Rui WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); 134e059c184SHuang Rui WRITE_ONCE(cpudata->lowest_nonlinear_perf, 135e059c184SHuang Rui cppc_perf.lowest_nonlinear_perf); 136e059c184SHuang Rui WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); 137e059c184SHuang Rui 138e059c184SHuang Rui return 0; 139e059c184SHuang Rui } 140e059c184SHuang Rui 141e059c184SHuang Rui DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); 142e059c184SHuang Rui 143e059c184SHuang Rui static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) 144e059c184SHuang Rui { 145e059c184SHuang Rui return static_call(amd_pstate_init_perf)(cpudata); 146e059c184SHuang Rui } 147e059c184SHuang Rui 148e059c184SHuang Rui static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, 149ec437d71SHuang Rui u32 des_perf, u32 max_perf, bool fast_switch) 150ec437d71SHuang Rui { 151ec437d71SHuang Rui if (fast_switch) 152ec437d71SHuang Rui wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); 153ec437d71SHuang Rui else 154ec437d71SHuang Rui wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, 155ec437d71SHuang Rui READ_ONCE(cpudata->cppc_req_cached)); 156ec437d71SHuang Rui } 157ec437d71SHuang Rui 158e059c184SHuang Rui static void cppc_update_perf(struct amd_cpudata *cpudata, 159e059c184SHuang Rui u32 min_perf, u32 des_perf, 160e059c184SHuang Rui u32 max_perf, bool fast_switch) 161e059c184SHuang Rui { 162e059c184SHuang Rui struct cppc_perf_ctrls perf_ctrls; 163e059c184SHuang Rui 164e059c184SHuang Rui perf_ctrls.max_perf = max_perf; 165e059c184SHuang Rui perf_ctrls.min_perf = min_perf; 166e059c184SHuang Rui perf_ctrls.desired_perf = des_perf; 167e059c184SHuang Rui 168e059c184SHuang Rui cppc_set_perf(cpudata->cpu, &perf_ctrls); 169e059c184SHuang Rui } 170e059c184SHuang Rui 171e059c184SHuang Rui DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); 172e059c184SHuang Rui 173e059c184SHuang Rui static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, 174e059c184SHuang Rui u32 min_perf, u32 des_perf, 175e059c184SHuang Rui u32 max_perf, bool fast_switch) 176e059c184SHuang Rui { 177e059c184SHuang Rui static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, 178e059c184SHuang Rui max_perf, fast_switch); 179e059c184SHuang Rui } 180e059c184SHuang Rui 18123c296fbSJinzhou Su static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) 18223c296fbSJinzhou Su { 18323c296fbSJinzhou Su u64 aperf, mperf, tsc; 18423c296fbSJinzhou Su unsigned long flags; 18523c296fbSJinzhou Su 18623c296fbSJinzhou Su local_irq_save(flags); 18723c296fbSJinzhou Su rdmsrl(MSR_IA32_APERF, aperf); 18823c296fbSJinzhou Su rdmsrl(MSR_IA32_MPERF, mperf); 18923c296fbSJinzhou Su tsc = rdtsc(); 19023c296fbSJinzhou Su 19123c296fbSJinzhou Su if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { 19223c296fbSJinzhou Su local_irq_restore(flags); 19323c296fbSJinzhou Su return false; 19423c296fbSJinzhou Su } 19523c296fbSJinzhou Su 19623c296fbSJinzhou Su local_irq_restore(flags); 19723c296fbSJinzhou Su 19823c296fbSJinzhou Su cpudata->cur.aperf = aperf; 19923c296fbSJinzhou Su cpudata->cur.mperf = mperf; 20023c296fbSJinzhou Su cpudata->cur.tsc = tsc; 20123c296fbSJinzhou Su cpudata->cur.aperf -= cpudata->prev.aperf; 20223c296fbSJinzhou Su cpudata->cur.mperf -= cpudata->prev.mperf; 20323c296fbSJinzhou Su cpudata->cur.tsc -= cpudata->prev.tsc; 20423c296fbSJinzhou Su 20523c296fbSJinzhou Su cpudata->prev.aperf = aperf; 20623c296fbSJinzhou Su cpudata->prev.mperf = mperf; 20723c296fbSJinzhou Su cpudata->prev.tsc = tsc; 20823c296fbSJinzhou Su 20923c296fbSJinzhou Su cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); 21023c296fbSJinzhou Su 21123c296fbSJinzhou Su return true; 21223c296fbSJinzhou Su } 21323c296fbSJinzhou Su 214ec437d71SHuang Rui static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, 215ec437d71SHuang Rui u32 des_perf, u32 max_perf, bool fast_switch) 216ec437d71SHuang Rui { 217ec437d71SHuang Rui u64 prev = READ_ONCE(cpudata->cppc_req_cached); 218ec437d71SHuang Rui u64 value = prev; 219ec437d71SHuang Rui 2200e9a8638SPerry Yuan des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); 221ec437d71SHuang Rui value &= ~AMD_CPPC_MIN_PERF(~0L); 222ec437d71SHuang Rui value |= AMD_CPPC_MIN_PERF(min_perf); 223ec437d71SHuang Rui 224ec437d71SHuang Rui value &= ~AMD_CPPC_DES_PERF(~0L); 225ec437d71SHuang Rui value |= AMD_CPPC_DES_PERF(des_perf); 226ec437d71SHuang Rui 227ec437d71SHuang Rui value &= ~AMD_CPPC_MAX_PERF(~0L); 228ec437d71SHuang Rui value |= AMD_CPPC_MAX_PERF(max_perf); 229ec437d71SHuang Rui 23023c296fbSJinzhou Su if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { 23123c296fbSJinzhou Su trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, 23223c296fbSJinzhou Su cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, 23360e10f89SHuang Rui cpudata->cpu, (value != prev), fast_switch); 23423c296fbSJinzhou Su } 23560e10f89SHuang Rui 236ec437d71SHuang Rui if (value == prev) 237ec437d71SHuang Rui return; 238ec437d71SHuang Rui 239ec437d71SHuang Rui WRITE_ONCE(cpudata->cppc_req_cached, value); 240ec437d71SHuang Rui 241ec437d71SHuang Rui amd_pstate_update_perf(cpudata, min_perf, des_perf, 242ec437d71SHuang Rui max_perf, fast_switch); 243ec437d71SHuang Rui } 244ec437d71SHuang Rui 245ec437d71SHuang Rui static int amd_pstate_verify(struct cpufreq_policy_data *policy) 246ec437d71SHuang Rui { 247ec437d71SHuang Rui cpufreq_verify_within_cpu_limits(policy); 248ec437d71SHuang Rui 249ec437d71SHuang Rui return 0; 250ec437d71SHuang Rui } 251ec437d71SHuang Rui 252ec437d71SHuang Rui static int amd_pstate_target(struct cpufreq_policy *policy, 253ec437d71SHuang Rui unsigned int target_freq, 254ec437d71SHuang Rui unsigned int relation) 255ec437d71SHuang Rui { 256ec437d71SHuang Rui struct cpufreq_freqs freqs; 257ec437d71SHuang Rui struct amd_cpudata *cpudata = policy->driver_data; 258ec437d71SHuang Rui unsigned long max_perf, min_perf, des_perf, cap_perf; 259ec437d71SHuang Rui 260ec437d71SHuang Rui if (!cpudata->max_freq) 261ec437d71SHuang Rui return -ENODEV; 262ec437d71SHuang Rui 263ec437d71SHuang Rui cap_perf = READ_ONCE(cpudata->highest_perf); 264b185c505SPerry Yuan min_perf = READ_ONCE(cpudata->lowest_perf); 265ec437d71SHuang Rui max_perf = cap_perf; 266ec437d71SHuang Rui 267ec437d71SHuang Rui freqs.old = policy->cur; 268ec437d71SHuang Rui freqs.new = target_freq; 269ec437d71SHuang Rui 270ec437d71SHuang Rui des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, 271ec437d71SHuang Rui cpudata->max_freq); 272ec437d71SHuang Rui 273ec437d71SHuang Rui cpufreq_freq_transition_begin(policy, &freqs); 274ec437d71SHuang Rui amd_pstate_update(cpudata, min_perf, des_perf, 275ec437d71SHuang Rui max_perf, false); 276ec437d71SHuang Rui cpufreq_freq_transition_end(policy, &freqs, false); 277ec437d71SHuang Rui 278ec437d71SHuang Rui return 0; 279ec437d71SHuang Rui } 280ec437d71SHuang Rui 2811d215f03SHuang Rui static void amd_pstate_adjust_perf(unsigned int cpu, 2821d215f03SHuang Rui unsigned long _min_perf, 2831d215f03SHuang Rui unsigned long target_perf, 2841d215f03SHuang Rui unsigned long capacity) 2851d215f03SHuang Rui { 2861d215f03SHuang Rui unsigned long max_perf, min_perf, des_perf, 2871d215f03SHuang Rui cap_perf, lowest_nonlinear_perf; 2881d215f03SHuang Rui struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 2891d215f03SHuang Rui struct amd_cpudata *cpudata = policy->driver_data; 2901d215f03SHuang Rui 2911d215f03SHuang Rui cap_perf = READ_ONCE(cpudata->highest_perf); 2921d215f03SHuang Rui lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); 2931d215f03SHuang Rui 2941d215f03SHuang Rui des_perf = cap_perf; 2951d215f03SHuang Rui if (target_perf < capacity) 2961d215f03SHuang Rui des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); 2971d215f03SHuang Rui 2981d215f03SHuang Rui min_perf = READ_ONCE(cpudata->highest_perf); 2991d215f03SHuang Rui if (_min_perf < capacity) 3001d215f03SHuang Rui min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); 3011d215f03SHuang Rui 3021d215f03SHuang Rui if (min_perf < lowest_nonlinear_perf) 3031d215f03SHuang Rui min_perf = lowest_nonlinear_perf; 3041d215f03SHuang Rui 3051d215f03SHuang Rui max_perf = cap_perf; 3061d215f03SHuang Rui if (max_perf < min_perf) 3071d215f03SHuang Rui max_perf = min_perf; 3081d215f03SHuang Rui 3091d215f03SHuang Rui amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); 3101d215f03SHuang Rui } 3111d215f03SHuang Rui 312ec437d71SHuang Rui static int amd_get_min_freq(struct amd_cpudata *cpudata) 313ec437d71SHuang Rui { 314ec437d71SHuang Rui struct cppc_perf_caps cppc_perf; 315ec437d71SHuang Rui 316ec437d71SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 317ec437d71SHuang Rui if (ret) 318ec437d71SHuang Rui return ret; 319ec437d71SHuang Rui 320ec437d71SHuang Rui /* Switch to khz */ 321ec437d71SHuang Rui return cppc_perf.lowest_freq * 1000; 322ec437d71SHuang Rui } 323ec437d71SHuang Rui 324ec437d71SHuang Rui static int amd_get_max_freq(struct amd_cpudata *cpudata) 325ec437d71SHuang Rui { 326ec437d71SHuang Rui struct cppc_perf_caps cppc_perf; 327ec437d71SHuang Rui u32 max_perf, max_freq, nominal_freq, nominal_perf; 328ec437d71SHuang Rui u64 boost_ratio; 329ec437d71SHuang Rui 330ec437d71SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 331ec437d71SHuang Rui if (ret) 332ec437d71SHuang Rui return ret; 333ec437d71SHuang Rui 334ec437d71SHuang Rui nominal_freq = cppc_perf.nominal_freq; 335ec437d71SHuang Rui nominal_perf = READ_ONCE(cpudata->nominal_perf); 336ec437d71SHuang Rui max_perf = READ_ONCE(cpudata->highest_perf); 337ec437d71SHuang Rui 338ec437d71SHuang Rui boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT, 339ec437d71SHuang Rui nominal_perf); 340ec437d71SHuang Rui 341ec437d71SHuang Rui max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT; 342ec437d71SHuang Rui 343ec437d71SHuang Rui /* Switch to khz */ 344ec437d71SHuang Rui return max_freq * 1000; 345ec437d71SHuang Rui } 346ec437d71SHuang Rui 347ec437d71SHuang Rui static int amd_get_nominal_freq(struct amd_cpudata *cpudata) 348ec437d71SHuang Rui { 349ec437d71SHuang Rui struct cppc_perf_caps cppc_perf; 350ec437d71SHuang Rui 351ec437d71SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 352ec437d71SHuang Rui if (ret) 353ec437d71SHuang Rui return ret; 354ec437d71SHuang Rui 355ec437d71SHuang Rui /* Switch to khz */ 356ec437d71SHuang Rui return cppc_perf.nominal_freq * 1000; 357ec437d71SHuang Rui } 358ec437d71SHuang Rui 359ec437d71SHuang Rui static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) 360ec437d71SHuang Rui { 361ec437d71SHuang Rui struct cppc_perf_caps cppc_perf; 362ec437d71SHuang Rui u32 lowest_nonlinear_freq, lowest_nonlinear_perf, 363ec437d71SHuang Rui nominal_freq, nominal_perf; 364ec437d71SHuang Rui u64 lowest_nonlinear_ratio; 365ec437d71SHuang Rui 366ec437d71SHuang Rui int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); 367ec437d71SHuang Rui if (ret) 368ec437d71SHuang Rui return ret; 369ec437d71SHuang Rui 370ec437d71SHuang Rui nominal_freq = cppc_perf.nominal_freq; 371ec437d71SHuang Rui nominal_perf = READ_ONCE(cpudata->nominal_perf); 372ec437d71SHuang Rui 373ec437d71SHuang Rui lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; 374ec437d71SHuang Rui 375ec437d71SHuang Rui lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, 376ec437d71SHuang Rui nominal_perf); 377ec437d71SHuang Rui 378ec437d71SHuang Rui lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT; 379ec437d71SHuang Rui 380ec437d71SHuang Rui /* Switch to khz */ 381ec437d71SHuang Rui return lowest_nonlinear_freq * 1000; 382ec437d71SHuang Rui } 383ec437d71SHuang Rui 38441271016SHuang Rui static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) 38541271016SHuang Rui { 38641271016SHuang Rui struct amd_cpudata *cpudata = policy->driver_data; 38741271016SHuang Rui int ret; 38841271016SHuang Rui 38941271016SHuang Rui if (!cpudata->boost_supported) { 39041271016SHuang Rui pr_err("Boost mode is not supported by this processor or SBIOS\n"); 39141271016SHuang Rui return -EINVAL; 39241271016SHuang Rui } 39341271016SHuang Rui 39441271016SHuang Rui if (state) 39541271016SHuang Rui policy->cpuinfo.max_freq = cpudata->max_freq; 39641271016SHuang Rui else 39741271016SHuang Rui policy->cpuinfo.max_freq = cpudata->nominal_freq; 39841271016SHuang Rui 39941271016SHuang Rui policy->max = policy->cpuinfo.max_freq; 40041271016SHuang Rui 40141271016SHuang Rui ret = freq_qos_update_request(&cpudata->req[1], 40241271016SHuang Rui policy->cpuinfo.max_freq); 40341271016SHuang Rui if (ret < 0) 40441271016SHuang Rui return ret; 40541271016SHuang Rui 40641271016SHuang Rui return 0; 40741271016SHuang Rui } 40841271016SHuang Rui 40941271016SHuang Rui static void amd_pstate_boost_init(struct amd_cpudata *cpudata) 41041271016SHuang Rui { 41141271016SHuang Rui u32 highest_perf, nominal_perf; 41241271016SHuang Rui 41341271016SHuang Rui highest_perf = READ_ONCE(cpudata->highest_perf); 41441271016SHuang Rui nominal_perf = READ_ONCE(cpudata->nominal_perf); 41541271016SHuang Rui 41641271016SHuang Rui if (highest_perf <= nominal_perf) 41741271016SHuang Rui return; 41841271016SHuang Rui 41941271016SHuang Rui cpudata->boost_supported = true; 42041271016SHuang Rui amd_pstate_driver.boost_enabled = true; 42141271016SHuang Rui } 42241271016SHuang Rui 423919f4557SWyes Karny static void amd_perf_ctl_reset(unsigned int cpu) 424919f4557SWyes Karny { 425919f4557SWyes Karny wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); 426919f4557SWyes Karny } 427919f4557SWyes Karny 428ec437d71SHuang Rui static int amd_pstate_cpu_init(struct cpufreq_policy *policy) 429ec437d71SHuang Rui { 430ec437d71SHuang Rui int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; 431ec437d71SHuang Rui struct device *dev; 432ec437d71SHuang Rui struct amd_cpudata *cpudata; 433ec437d71SHuang Rui 434919f4557SWyes Karny /* 435919f4557SWyes Karny * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, 436919f4557SWyes Karny * which is ideal for initialization process. 437919f4557SWyes Karny */ 438919f4557SWyes Karny amd_perf_ctl_reset(policy->cpu); 439ec437d71SHuang Rui dev = get_cpu_device(policy->cpu); 440ec437d71SHuang Rui if (!dev) 441ec437d71SHuang Rui return -ENODEV; 442ec437d71SHuang Rui 443ec437d71SHuang Rui cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); 444ec437d71SHuang Rui if (!cpudata) 445ec437d71SHuang Rui return -ENOMEM; 446ec437d71SHuang Rui 447ec437d71SHuang Rui cpudata->cpu = policy->cpu; 448ec437d71SHuang Rui 449ec437d71SHuang Rui ret = amd_pstate_init_perf(cpudata); 450ec437d71SHuang Rui if (ret) 45141271016SHuang Rui goto free_cpudata1; 452ec437d71SHuang Rui 453ec437d71SHuang Rui min_freq = amd_get_min_freq(cpudata); 454ec437d71SHuang Rui max_freq = amd_get_max_freq(cpudata); 455ec437d71SHuang Rui nominal_freq = amd_get_nominal_freq(cpudata); 456ec437d71SHuang Rui lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); 457ec437d71SHuang Rui 458ec437d71SHuang Rui if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { 459ec437d71SHuang Rui dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", 460ec437d71SHuang Rui min_freq, max_freq); 461ec437d71SHuang Rui ret = -EINVAL; 46241271016SHuang Rui goto free_cpudata1; 463ec437d71SHuang Rui } 464ec437d71SHuang Rui 465ec437d71SHuang Rui policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; 466ec437d71SHuang Rui policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; 467ec437d71SHuang Rui 468ec437d71SHuang Rui policy->min = min_freq; 469ec437d71SHuang Rui policy->max = max_freq; 470ec437d71SHuang Rui 471ec437d71SHuang Rui policy->cpuinfo.min_freq = min_freq; 472ec437d71SHuang Rui policy->cpuinfo.max_freq = max_freq; 473ec437d71SHuang Rui 474ec437d71SHuang Rui /* It will be updated by governor */ 475ec437d71SHuang Rui policy->cur = policy->cpuinfo.min_freq; 476ec437d71SHuang Rui 477e059c184SHuang Rui if (boot_cpu_has(X86_FEATURE_CPPC)) 4781d215f03SHuang Rui policy->fast_switch_possible = true; 4791d215f03SHuang Rui 48041271016SHuang Rui ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], 48141271016SHuang Rui FREQ_QOS_MIN, policy->cpuinfo.min_freq); 48241271016SHuang Rui if (ret < 0) { 48341271016SHuang Rui dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); 48441271016SHuang Rui goto free_cpudata1; 48541271016SHuang Rui } 48641271016SHuang Rui 48741271016SHuang Rui ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], 48841271016SHuang Rui FREQ_QOS_MAX, policy->cpuinfo.max_freq); 48941271016SHuang Rui if (ret < 0) { 49041271016SHuang Rui dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); 49141271016SHuang Rui goto free_cpudata2; 49241271016SHuang Rui } 49341271016SHuang Rui 494ec437d71SHuang Rui /* Initial processor data capability frequencies */ 495ec437d71SHuang Rui cpudata->max_freq = max_freq; 496ec437d71SHuang Rui cpudata->min_freq = min_freq; 497ec437d71SHuang Rui cpudata->nominal_freq = nominal_freq; 498ec437d71SHuang Rui cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; 499ec437d71SHuang Rui 500ec437d71SHuang Rui policy->driver_data = cpudata; 501ec437d71SHuang Rui 50241271016SHuang Rui amd_pstate_boost_init(cpudata); 50341271016SHuang Rui 504ec437d71SHuang Rui return 0; 505ec437d71SHuang Rui 50641271016SHuang Rui free_cpudata2: 50741271016SHuang Rui freq_qos_remove_request(&cpudata->req[0]); 50841271016SHuang Rui free_cpudata1: 509ec437d71SHuang Rui kfree(cpudata); 510ec437d71SHuang Rui return ret; 511ec437d71SHuang Rui } 512ec437d71SHuang Rui 513ec437d71SHuang Rui static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) 514ec437d71SHuang Rui { 5154f59540cSPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 516ec437d71SHuang Rui 51741271016SHuang Rui freq_qos_remove_request(&cpudata->req[1]); 51841271016SHuang Rui freq_qos_remove_request(&cpudata->req[0]); 519ec437d71SHuang Rui kfree(cpudata); 520ec437d71SHuang Rui 521ec437d71SHuang Rui return 0; 522ec437d71SHuang Rui } 523ec437d71SHuang Rui 524b376471fSJinzhou Su static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) 525b376471fSJinzhou Su { 526b376471fSJinzhou Su int ret; 527b376471fSJinzhou Su 528b376471fSJinzhou Su ret = amd_pstate_enable(true); 529b376471fSJinzhou Su if (ret) 530b376471fSJinzhou Su pr_err("failed to enable amd-pstate during resume, return %d\n", ret); 531b376471fSJinzhou Su 532b376471fSJinzhou Su return ret; 533b376471fSJinzhou Su } 534b376471fSJinzhou Su 535b376471fSJinzhou Su static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) 536b376471fSJinzhou Su { 537b376471fSJinzhou Su int ret; 538b376471fSJinzhou Su 539b376471fSJinzhou Su ret = amd_pstate_enable(false); 540b376471fSJinzhou Su if (ret) 541b376471fSJinzhou Su pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); 542b376471fSJinzhou Su 543b376471fSJinzhou Su return ret; 544b376471fSJinzhou Su } 545b376471fSJinzhou Su 546ec4e3326SHuang Rui /* Sysfs attributes */ 547ec4e3326SHuang Rui 548ec4e3326SHuang Rui /* 549ec4e3326SHuang Rui * This frequency is to indicate the maximum hardware frequency. 550ec4e3326SHuang Rui * If boost is not active but supported, the frequency will be larger than the 551ec4e3326SHuang Rui * one in cpuinfo. 552ec4e3326SHuang Rui */ 553ec4e3326SHuang Rui static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, 554ec4e3326SHuang Rui char *buf) 555ec4e3326SHuang Rui { 556ec4e3326SHuang Rui int max_freq; 5574f59540cSPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 558ec4e3326SHuang Rui 559ec4e3326SHuang Rui max_freq = amd_get_max_freq(cpudata); 560ec4e3326SHuang Rui if (max_freq < 0) 561ec4e3326SHuang Rui return max_freq; 562ec4e3326SHuang Rui 563ec4e3326SHuang Rui return sprintf(&buf[0], "%u\n", max_freq); 564ec4e3326SHuang Rui } 565ec4e3326SHuang Rui 566ec4e3326SHuang Rui static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, 567ec4e3326SHuang Rui char *buf) 568ec4e3326SHuang Rui { 569ec4e3326SHuang Rui int freq; 5704f59540cSPerry Yuan struct amd_cpudata *cpudata = policy->driver_data; 571ec4e3326SHuang Rui 572ec4e3326SHuang Rui freq = amd_get_lowest_nonlinear_freq(cpudata); 573ec4e3326SHuang Rui if (freq < 0) 574ec4e3326SHuang Rui return freq; 575ec4e3326SHuang Rui 576ec4e3326SHuang Rui return sprintf(&buf[0], "%u\n", freq); 577ec4e3326SHuang Rui } 578ec4e3326SHuang Rui 5793ad7fde1SHuang Rui /* 5803ad7fde1SHuang Rui * In some of ASICs, the highest_perf is not the one in the _CPC table, so we 5813ad7fde1SHuang Rui * need to expose it to sysfs. 5823ad7fde1SHuang Rui */ 5833ad7fde1SHuang Rui static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, 5843ad7fde1SHuang Rui char *buf) 5853ad7fde1SHuang Rui { 5863ad7fde1SHuang Rui u32 perf; 5873ad7fde1SHuang Rui struct amd_cpudata *cpudata = policy->driver_data; 5883ad7fde1SHuang Rui 5893ad7fde1SHuang Rui perf = READ_ONCE(cpudata->highest_perf); 5903ad7fde1SHuang Rui 5913ad7fde1SHuang Rui return sprintf(&buf[0], "%u\n", perf); 5923ad7fde1SHuang Rui } 5933ad7fde1SHuang Rui 594ec4e3326SHuang Rui cpufreq_freq_attr_ro(amd_pstate_max_freq); 595ec4e3326SHuang Rui cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); 596ec4e3326SHuang Rui 5973ad7fde1SHuang Rui cpufreq_freq_attr_ro(amd_pstate_highest_perf); 5983ad7fde1SHuang Rui 599ec4e3326SHuang Rui static struct freq_attr *amd_pstate_attr[] = { 600ec4e3326SHuang Rui &amd_pstate_max_freq, 601ec4e3326SHuang Rui &amd_pstate_lowest_nonlinear_freq, 6023ad7fde1SHuang Rui &amd_pstate_highest_perf, 603ec4e3326SHuang Rui NULL, 604ec4e3326SHuang Rui }; 605ec4e3326SHuang Rui 606ec437d71SHuang Rui static struct cpufreq_driver amd_pstate_driver = { 607ec437d71SHuang Rui .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, 608ec437d71SHuang Rui .verify = amd_pstate_verify, 609ec437d71SHuang Rui .target = amd_pstate_target, 610ec437d71SHuang Rui .init = amd_pstate_cpu_init, 611ec437d71SHuang Rui .exit = amd_pstate_cpu_exit, 612b376471fSJinzhou Su .suspend = amd_pstate_cpu_suspend, 613b376471fSJinzhou Su .resume = amd_pstate_cpu_resume, 61441271016SHuang Rui .set_boost = amd_pstate_set_boost, 615ec437d71SHuang Rui .name = "amd-pstate", 616ec4e3326SHuang Rui .attr = amd_pstate_attr, 617ec437d71SHuang Rui }; 618ec437d71SHuang Rui 619ec437d71SHuang Rui static int __init amd_pstate_init(void) 620ec437d71SHuang Rui { 621ec437d71SHuang Rui int ret; 622ec437d71SHuang Rui 623ec437d71SHuang Rui if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 624ec437d71SHuang Rui return -ENODEV; 625*202e683dSPerry Yuan /* 626*202e683dSPerry Yuan * by default the pstate driver is disabled to load 627*202e683dSPerry Yuan * enable the amd_pstate passive mode driver explicitly 628*202e683dSPerry Yuan * with amd_pstate=passive in kernel command line 629*202e683dSPerry Yuan */ 630*202e683dSPerry Yuan if (!cppc_load) { 631*202e683dSPerry Yuan pr_debug("driver load is disabled, boot with amd_pstate=passive to enable this\n"); 632*202e683dSPerry Yuan return -ENODEV; 633*202e683dSPerry Yuan } 634ec437d71SHuang Rui 635ec437d71SHuang Rui if (!acpi_cpc_valid()) { 636a2a9d185SPerry Yuan pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); 637ec437d71SHuang Rui return -ENODEV; 638ec437d71SHuang Rui } 639ec437d71SHuang Rui 640ec437d71SHuang Rui /* don't keep reloading if cpufreq_driver exists */ 641ec437d71SHuang Rui if (cpufreq_get_current_driver()) 642ec437d71SHuang Rui return -EEXIST; 643ec437d71SHuang Rui 644ec437d71SHuang Rui /* capability check */ 645e059c184SHuang Rui if (boot_cpu_has(X86_FEATURE_CPPC)) { 646e059c184SHuang Rui pr_debug("AMD CPPC MSR based functionality is supported\n"); 647e059c184SHuang Rui amd_pstate_driver.adjust_perf = amd_pstate_adjust_perf; 648*202e683dSPerry Yuan } else { 649*202e683dSPerry Yuan pr_debug("AMD CPPC shared memory based functionality is supported\n"); 650e059c184SHuang Rui static_call_update(amd_pstate_enable, cppc_enable); 651e059c184SHuang Rui static_call_update(amd_pstate_init_perf, cppc_init_perf); 652e059c184SHuang Rui static_call_update(amd_pstate_update_perf, cppc_update_perf); 653ec437d71SHuang Rui } 654ec437d71SHuang Rui 655ec437d71SHuang Rui /* enable amd pstate feature */ 656ec437d71SHuang Rui ret = amd_pstate_enable(true); 657ec437d71SHuang Rui if (ret) { 658ec437d71SHuang Rui pr_err("failed to enable amd-pstate with return %d\n", ret); 659ec437d71SHuang Rui return ret; 660ec437d71SHuang Rui } 661ec437d71SHuang Rui 662ec437d71SHuang Rui ret = cpufreq_register_driver(&amd_pstate_driver); 663ec437d71SHuang Rui if (ret) 664ec437d71SHuang Rui pr_err("failed to register amd_pstate_driver with return %d\n", 665ec437d71SHuang Rui ret); 666ec437d71SHuang Rui 667ec437d71SHuang Rui return ret; 668ec437d71SHuang Rui } 669456ca88dSPerry Yuan device_initcall(amd_pstate_init); 670ec437d71SHuang Rui 671*202e683dSPerry Yuan static int __init amd_pstate_param(char *str) 672*202e683dSPerry Yuan { 673*202e683dSPerry Yuan if (!str) 674*202e683dSPerry Yuan return -EINVAL; 675*202e683dSPerry Yuan 676*202e683dSPerry Yuan if (!strcmp(str, "disable")) { 677*202e683dSPerry Yuan cppc_load = 0; 678*202e683dSPerry Yuan pr_info("driver is explicitly disabled\n"); 679*202e683dSPerry Yuan } else if (!strcmp(str, "passive")) 680*202e683dSPerry Yuan cppc_load = 1; 681*202e683dSPerry Yuan 682*202e683dSPerry Yuan return 0; 683*202e683dSPerry Yuan } 684*202e683dSPerry Yuan early_param("amd_pstate", amd_pstate_param); 685*202e683dSPerry Yuan 686ec437d71SHuang Rui MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); 687ec437d71SHuang Rui MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); 688ec437d71SHuang Rui MODULE_LICENSE("GPL"); 689