1*6286bbb4SHector Martin // SPDX-License-Identifier: GPL-2.0-only 2*6286bbb4SHector Martin /* 3*6286bbb4SHector Martin * Apple SoC CPU cluster performance state driver 4*6286bbb4SHector Martin * 5*6286bbb4SHector Martin * Copyright The Asahi Linux Contributors 6*6286bbb4SHector Martin * 7*6286bbb4SHector Martin * Based on scpi-cpufreq.c 8*6286bbb4SHector Martin */ 9*6286bbb4SHector Martin 10*6286bbb4SHector Martin #include <linux/bitfield.h> 11*6286bbb4SHector Martin #include <linux/bitops.h> 12*6286bbb4SHector Martin #include <linux/cpu.h> 13*6286bbb4SHector Martin #include <linux/cpufreq.h> 14*6286bbb4SHector Martin #include <linux/cpumask.h> 15*6286bbb4SHector Martin #include <linux/delay.h> 16*6286bbb4SHector Martin #include <linux/err.h> 17*6286bbb4SHector Martin #include <linux/io.h> 18*6286bbb4SHector Martin #include <linux/iopoll.h> 19*6286bbb4SHector Martin #include <linux/module.h> 20*6286bbb4SHector Martin #include <linux/of.h> 21*6286bbb4SHector Martin #include <linux/of_address.h> 22*6286bbb4SHector Martin #include <linux/pm_opp.h> 23*6286bbb4SHector Martin #include <linux/slab.h> 24*6286bbb4SHector Martin 25*6286bbb4SHector Martin #define APPLE_DVFS_CMD 0x20 26*6286bbb4SHector Martin #define APPLE_DVFS_CMD_BUSY BIT(31) 27*6286bbb4SHector Martin #define APPLE_DVFS_CMD_SET BIT(25) 28*6286bbb4SHector Martin #define APPLE_DVFS_CMD_PS2 GENMASK(16, 12) 29*6286bbb4SHector Martin #define APPLE_DVFS_CMD_PS1 GENMASK(4, 0) 30*6286bbb4SHector Martin 31*6286bbb4SHector Martin /* Same timebase as CPU counter (24MHz) */ 32*6286bbb4SHector Martin #define APPLE_DVFS_LAST_CHG_TIME 0x38 33*6286bbb4SHector Martin 34*6286bbb4SHector Martin /* 35*6286bbb4SHector Martin * Apple ran out of bits and had to shift this in T8112... 36*6286bbb4SHector Martin */ 37*6286bbb4SHector Martin #define APPLE_DVFS_STATUS 0x50 38*6286bbb4SHector Martin #define APPLE_DVFS_STATUS_CUR_PS_T8103 GENMASK(7, 4) 39*6286bbb4SHector Martin #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103 4 40*6286bbb4SHector Martin #define APPLE_DVFS_STATUS_TGT_PS_T8103 GENMASK(3, 0) 41*6286bbb4SHector Martin #define APPLE_DVFS_STATUS_CUR_PS_T8112 GENMASK(9, 5) 42*6286bbb4SHector Martin #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112 5 43*6286bbb4SHector Martin #define APPLE_DVFS_STATUS_TGT_PS_T8112 GENMASK(4, 0) 44*6286bbb4SHector Martin 45*6286bbb4SHector Martin /* 46*6286bbb4SHector Martin * Div is +1, base clock is 12MHz on existing SoCs. 47*6286bbb4SHector Martin * For documentation purposes. We use the OPP table to 48*6286bbb4SHector Martin * get the frequency. 49*6286bbb4SHector Martin */ 50*6286bbb4SHector Martin #define APPLE_DVFS_PLL_STATUS 0xc0 51*6286bbb4SHector Martin #define APPLE_DVFS_PLL_FACTOR 0xc8 52*6286bbb4SHector Martin #define APPLE_DVFS_PLL_FACTOR_MULT GENMASK(31, 16) 53*6286bbb4SHector Martin #define APPLE_DVFS_PLL_FACTOR_DIV GENMASK(15, 0) 54*6286bbb4SHector Martin 55*6286bbb4SHector Martin #define APPLE_DVFS_TRANSITION_TIMEOUT 100 56*6286bbb4SHector Martin 57*6286bbb4SHector Martin struct apple_soc_cpufreq_info { 58*6286bbb4SHector Martin u64 max_pstate; 59*6286bbb4SHector Martin u64 cur_pstate_mask; 60*6286bbb4SHector Martin u64 cur_pstate_shift; 61*6286bbb4SHector Martin }; 62*6286bbb4SHector Martin 63*6286bbb4SHector Martin struct apple_cpu_priv { 64*6286bbb4SHector Martin struct device *cpu_dev; 65*6286bbb4SHector Martin void __iomem *reg_base; 66*6286bbb4SHector Martin const struct apple_soc_cpufreq_info *info; 67*6286bbb4SHector Martin }; 68*6286bbb4SHector Martin 69*6286bbb4SHector Martin static struct cpufreq_driver apple_soc_cpufreq_driver; 70*6286bbb4SHector Martin 71*6286bbb4SHector Martin static const struct apple_soc_cpufreq_info soc_t8103_info = { 72*6286bbb4SHector Martin .max_pstate = 15, 73*6286bbb4SHector Martin .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103, 74*6286bbb4SHector Martin .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103, 75*6286bbb4SHector Martin }; 76*6286bbb4SHector Martin 77*6286bbb4SHector Martin static const struct apple_soc_cpufreq_info soc_t8112_info = { 78*6286bbb4SHector Martin .max_pstate = 31, 79*6286bbb4SHector Martin .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112, 80*6286bbb4SHector Martin .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112, 81*6286bbb4SHector Martin }; 82*6286bbb4SHector Martin 83*6286bbb4SHector Martin static const struct apple_soc_cpufreq_info soc_default_info = { 84*6286bbb4SHector Martin .max_pstate = 15, 85*6286bbb4SHector Martin .cur_pstate_mask = 0, /* fallback */ 86*6286bbb4SHector Martin }; 87*6286bbb4SHector Martin 88*6286bbb4SHector Martin static const struct of_device_id apple_soc_cpufreq_of_match[] = { 89*6286bbb4SHector Martin { 90*6286bbb4SHector Martin .compatible = "apple,t8103-cluster-cpufreq", 91*6286bbb4SHector Martin .data = &soc_t8103_info, 92*6286bbb4SHector Martin }, 93*6286bbb4SHector Martin { 94*6286bbb4SHector Martin .compatible = "apple,t8112-cluster-cpufreq", 95*6286bbb4SHector Martin .data = &soc_t8112_info, 96*6286bbb4SHector Martin }, 97*6286bbb4SHector Martin { 98*6286bbb4SHector Martin .compatible = "apple,cluster-cpufreq", 99*6286bbb4SHector Martin .data = &soc_default_info, 100*6286bbb4SHector Martin }, 101*6286bbb4SHector Martin {} 102*6286bbb4SHector Martin }; 103*6286bbb4SHector Martin 104*6286bbb4SHector Martin static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu) 105*6286bbb4SHector Martin { 106*6286bbb4SHector Martin struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu); 107*6286bbb4SHector Martin struct apple_cpu_priv *priv = policy->driver_data; 108*6286bbb4SHector Martin struct cpufreq_frequency_table *p; 109*6286bbb4SHector Martin unsigned int pstate; 110*6286bbb4SHector Martin 111*6286bbb4SHector Martin if (priv->info->cur_pstate_mask) { 112*6286bbb4SHector Martin u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_STATUS); 113*6286bbb4SHector Martin 114*6286bbb4SHector Martin pstate = (reg & priv->info->cur_pstate_mask) >> priv->info->cur_pstate_shift; 115*6286bbb4SHector Martin } else { 116*6286bbb4SHector Martin /* 117*6286bbb4SHector Martin * For the fallback case we might not know the layout of DVFS_STATUS, 118*6286bbb4SHector Martin * so just use the command register value (which ignores boost limitations). 119*6286bbb4SHector Martin */ 120*6286bbb4SHector Martin u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD); 121*6286bbb4SHector Martin 122*6286bbb4SHector Martin pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg); 123*6286bbb4SHector Martin } 124*6286bbb4SHector Martin 125*6286bbb4SHector Martin cpufreq_for_each_valid_entry(p, policy->freq_table) 126*6286bbb4SHector Martin if (p->driver_data == pstate) 127*6286bbb4SHector Martin return p->frequency; 128*6286bbb4SHector Martin 129*6286bbb4SHector Martin dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n", 130*6286bbb4SHector Martin pstate); 131*6286bbb4SHector Martin return 0; 132*6286bbb4SHector Martin } 133*6286bbb4SHector Martin 134*6286bbb4SHector Martin static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy, 135*6286bbb4SHector Martin unsigned int index) 136*6286bbb4SHector Martin { 137*6286bbb4SHector Martin struct apple_cpu_priv *priv = policy->driver_data; 138*6286bbb4SHector Martin unsigned int pstate = policy->freq_table[index].driver_data; 139*6286bbb4SHector Martin u64 reg; 140*6286bbb4SHector Martin 141*6286bbb4SHector Martin /* Fallback for newer SoCs */ 142*6286bbb4SHector Martin if (index > priv->info->max_pstate) 143*6286bbb4SHector Martin index = priv->info->max_pstate; 144*6286bbb4SHector Martin 145*6286bbb4SHector Martin if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg, 146*6286bbb4SHector Martin !(reg & APPLE_DVFS_CMD_BUSY), 2, 147*6286bbb4SHector Martin APPLE_DVFS_TRANSITION_TIMEOUT)) { 148*6286bbb4SHector Martin return -EIO; 149*6286bbb4SHector Martin } 150*6286bbb4SHector Martin 151*6286bbb4SHector Martin reg &= ~(APPLE_DVFS_CMD_PS1 | APPLE_DVFS_CMD_PS2); 152*6286bbb4SHector Martin reg |= FIELD_PREP(APPLE_DVFS_CMD_PS1, pstate); 153*6286bbb4SHector Martin reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate); 154*6286bbb4SHector Martin reg |= APPLE_DVFS_CMD_SET; 155*6286bbb4SHector Martin 156*6286bbb4SHector Martin writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD); 157*6286bbb4SHector Martin 158*6286bbb4SHector Martin return 0; 159*6286bbb4SHector Martin } 160*6286bbb4SHector Martin 161*6286bbb4SHector Martin static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy, 162*6286bbb4SHector Martin unsigned int target_freq) 163*6286bbb4SHector Martin { 164*6286bbb4SHector Martin if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0) 165*6286bbb4SHector Martin return 0; 166*6286bbb4SHector Martin 167*6286bbb4SHector Martin return policy->freq_table[policy->cached_resolved_idx].frequency; 168*6286bbb4SHector Martin } 169*6286bbb4SHector Martin 170*6286bbb4SHector Martin static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy, 171*6286bbb4SHector Martin void __iomem **reg_base, 172*6286bbb4SHector Martin const struct apple_soc_cpufreq_info **info) 173*6286bbb4SHector Martin { 174*6286bbb4SHector Martin struct of_phandle_args args; 175*6286bbb4SHector Martin const struct of_device_id *match; 176*6286bbb4SHector Martin int ret = 0; 177*6286bbb4SHector Martin 178*6286bbb4SHector Martin ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains", 179*6286bbb4SHector Martin "#performance-domain-cells", 180*6286bbb4SHector Martin policy->cpus, &args); 181*6286bbb4SHector Martin if (ret < 0) 182*6286bbb4SHector Martin return ret; 183*6286bbb4SHector Martin 184*6286bbb4SHector Martin match = of_match_node(apple_soc_cpufreq_of_match, args.np); 185*6286bbb4SHector Martin of_node_put(args.np); 186*6286bbb4SHector Martin if (!match) 187*6286bbb4SHector Martin return -ENODEV; 188*6286bbb4SHector Martin 189*6286bbb4SHector Martin *info = match->data; 190*6286bbb4SHector Martin 191*6286bbb4SHector Martin *reg_base = of_iomap(args.np, 0); 192*6286bbb4SHector Martin if (IS_ERR(*reg_base)) 193*6286bbb4SHector Martin return PTR_ERR(*reg_base); 194*6286bbb4SHector Martin 195*6286bbb4SHector Martin return 0; 196*6286bbb4SHector Martin } 197*6286bbb4SHector Martin 198*6286bbb4SHector Martin static struct freq_attr *apple_soc_cpufreq_hw_attr[] = { 199*6286bbb4SHector Martin &cpufreq_freq_attr_scaling_available_freqs, 200*6286bbb4SHector Martin NULL, /* Filled in below if boost is enabled */ 201*6286bbb4SHector Martin NULL, 202*6286bbb4SHector Martin }; 203*6286bbb4SHector Martin 204*6286bbb4SHector Martin static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) 205*6286bbb4SHector Martin { 206*6286bbb4SHector Martin int ret, i; 207*6286bbb4SHector Martin unsigned int transition_latency; 208*6286bbb4SHector Martin void __iomem *reg_base; 209*6286bbb4SHector Martin struct device *cpu_dev; 210*6286bbb4SHector Martin struct apple_cpu_priv *priv; 211*6286bbb4SHector Martin const struct apple_soc_cpufreq_info *info; 212*6286bbb4SHector Martin struct cpufreq_frequency_table *freq_table; 213*6286bbb4SHector Martin 214*6286bbb4SHector Martin cpu_dev = get_cpu_device(policy->cpu); 215*6286bbb4SHector Martin if (!cpu_dev) { 216*6286bbb4SHector Martin pr_err("failed to get cpu%d device\n", policy->cpu); 217*6286bbb4SHector Martin return -ENODEV; 218*6286bbb4SHector Martin } 219*6286bbb4SHector Martin 220*6286bbb4SHector Martin ret = dev_pm_opp_of_add_table(cpu_dev); 221*6286bbb4SHector Martin if (ret < 0) { 222*6286bbb4SHector Martin dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); 223*6286bbb4SHector Martin return ret; 224*6286bbb4SHector Martin } 225*6286bbb4SHector Martin 226*6286bbb4SHector Martin ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info); 227*6286bbb4SHector Martin if (ret) { 228*6286bbb4SHector Martin dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret); 229*6286bbb4SHector Martin return ret; 230*6286bbb4SHector Martin } 231*6286bbb4SHector Martin 232*6286bbb4SHector Martin ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); 233*6286bbb4SHector Martin if (ret) { 234*6286bbb4SHector Martin dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret); 235*6286bbb4SHector Martin goto out_iounmap; 236*6286bbb4SHector Martin } 237*6286bbb4SHector Martin 238*6286bbb4SHector Martin ret = dev_pm_opp_get_opp_count(cpu_dev); 239*6286bbb4SHector Martin if (ret <= 0) { 240*6286bbb4SHector Martin dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n"); 241*6286bbb4SHector Martin ret = -EPROBE_DEFER; 242*6286bbb4SHector Martin goto out_free_opp; 243*6286bbb4SHector Martin } 244*6286bbb4SHector Martin 245*6286bbb4SHector Martin priv = kzalloc(sizeof(*priv), GFP_KERNEL); 246*6286bbb4SHector Martin if (!priv) { 247*6286bbb4SHector Martin ret = -ENOMEM; 248*6286bbb4SHector Martin goto out_free_opp; 249*6286bbb4SHector Martin } 250*6286bbb4SHector Martin 251*6286bbb4SHector Martin ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); 252*6286bbb4SHector Martin if (ret) { 253*6286bbb4SHector Martin dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); 254*6286bbb4SHector Martin goto out_free_priv; 255*6286bbb4SHector Martin } 256*6286bbb4SHector Martin 257*6286bbb4SHector Martin /* Get OPP levels (p-state indexes) and stash them in driver_data */ 258*6286bbb4SHector Martin for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { 259*6286bbb4SHector Martin unsigned long rate = freq_table[i].frequency * 1000 + 999; 260*6286bbb4SHector Martin struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate); 261*6286bbb4SHector Martin 262*6286bbb4SHector Martin if (IS_ERR(opp)) { 263*6286bbb4SHector Martin ret = PTR_ERR(opp); 264*6286bbb4SHector Martin goto out_free_cpufreq_table; 265*6286bbb4SHector Martin } 266*6286bbb4SHector Martin freq_table[i].driver_data = dev_pm_opp_get_level(opp); 267*6286bbb4SHector Martin dev_pm_opp_put(opp); 268*6286bbb4SHector Martin } 269*6286bbb4SHector Martin 270*6286bbb4SHector Martin priv->cpu_dev = cpu_dev; 271*6286bbb4SHector Martin priv->reg_base = reg_base; 272*6286bbb4SHector Martin priv->info = info; 273*6286bbb4SHector Martin policy->driver_data = priv; 274*6286bbb4SHector Martin policy->freq_table = freq_table; 275*6286bbb4SHector Martin 276*6286bbb4SHector Martin transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev); 277*6286bbb4SHector Martin if (!transition_latency) 278*6286bbb4SHector Martin transition_latency = CPUFREQ_ETERNAL; 279*6286bbb4SHector Martin 280*6286bbb4SHector Martin policy->cpuinfo.transition_latency = transition_latency; 281*6286bbb4SHector Martin policy->dvfs_possible_from_any_cpu = true; 282*6286bbb4SHector Martin policy->fast_switch_possible = true; 283*6286bbb4SHector Martin 284*6286bbb4SHector Martin if (policy_has_boost_freq(policy)) { 285*6286bbb4SHector Martin ret = cpufreq_enable_boost_support(); 286*6286bbb4SHector Martin if (ret) { 287*6286bbb4SHector Martin dev_warn(cpu_dev, "failed to enable boost: %d\n", ret); 288*6286bbb4SHector Martin } else { 289*6286bbb4SHector Martin apple_soc_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs; 290*6286bbb4SHector Martin apple_soc_cpufreq_driver.boost_enabled = true; 291*6286bbb4SHector Martin } 292*6286bbb4SHector Martin } 293*6286bbb4SHector Martin 294*6286bbb4SHector Martin return 0; 295*6286bbb4SHector Martin 296*6286bbb4SHector Martin out_free_cpufreq_table: 297*6286bbb4SHector Martin dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); 298*6286bbb4SHector Martin out_free_priv: 299*6286bbb4SHector Martin kfree(priv); 300*6286bbb4SHector Martin out_free_opp: 301*6286bbb4SHector Martin dev_pm_opp_remove_all_dynamic(cpu_dev); 302*6286bbb4SHector Martin out_iounmap: 303*6286bbb4SHector Martin iounmap(reg_base); 304*6286bbb4SHector Martin return ret; 305*6286bbb4SHector Martin } 306*6286bbb4SHector Martin 307*6286bbb4SHector Martin static int apple_soc_cpufreq_exit(struct cpufreq_policy *policy) 308*6286bbb4SHector Martin { 309*6286bbb4SHector Martin struct apple_cpu_priv *priv = policy->driver_data; 310*6286bbb4SHector Martin 311*6286bbb4SHector Martin dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); 312*6286bbb4SHector Martin dev_pm_opp_remove_all_dynamic(priv->cpu_dev); 313*6286bbb4SHector Martin iounmap(priv->reg_base); 314*6286bbb4SHector Martin kfree(priv); 315*6286bbb4SHector Martin 316*6286bbb4SHector Martin return 0; 317*6286bbb4SHector Martin } 318*6286bbb4SHector Martin 319*6286bbb4SHector Martin static struct cpufreq_driver apple_soc_cpufreq_driver = { 320*6286bbb4SHector Martin .name = "apple-cpufreq", 321*6286bbb4SHector Martin .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY | 322*6286bbb4SHector Martin CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV, 323*6286bbb4SHector Martin .verify = cpufreq_generic_frequency_table_verify, 324*6286bbb4SHector Martin .attr = cpufreq_generic_attr, 325*6286bbb4SHector Martin .get = apple_soc_cpufreq_get_rate, 326*6286bbb4SHector Martin .init = apple_soc_cpufreq_init, 327*6286bbb4SHector Martin .exit = apple_soc_cpufreq_exit, 328*6286bbb4SHector Martin .target_index = apple_soc_cpufreq_set_target, 329*6286bbb4SHector Martin .fast_switch = apple_soc_cpufreq_fast_switch, 330*6286bbb4SHector Martin .register_em = cpufreq_register_em_with_opp, 331*6286bbb4SHector Martin .attr = apple_soc_cpufreq_hw_attr, 332*6286bbb4SHector Martin }; 333*6286bbb4SHector Martin 334*6286bbb4SHector Martin static int __init apple_soc_cpufreq_module_init(void) 335*6286bbb4SHector Martin { 336*6286bbb4SHector Martin if (!of_machine_is_compatible("apple,arm-platform")) 337*6286bbb4SHector Martin return -ENODEV; 338*6286bbb4SHector Martin 339*6286bbb4SHector Martin return cpufreq_register_driver(&apple_soc_cpufreq_driver); 340*6286bbb4SHector Martin } 341*6286bbb4SHector Martin module_init(apple_soc_cpufreq_module_init); 342*6286bbb4SHector Martin 343*6286bbb4SHector Martin static void __exit apple_soc_cpufreq_module_exit(void) 344*6286bbb4SHector Martin { 345*6286bbb4SHector Martin cpufreq_unregister_driver(&apple_soc_cpufreq_driver); 346*6286bbb4SHector Martin } 347*6286bbb4SHector Martin module_exit(apple_soc_cpufreq_module_exit); 348*6286bbb4SHector Martin 349*6286bbb4SHector Martin MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match); 350*6286bbb4SHector Martin MODULE_AUTHOR("Hector Martin <marcan@marcan.st>"); 351*6286bbb4SHector Martin MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver"); 352*6286bbb4SHector Martin MODULE_LICENSE("GPL"); 353