16286bbb4SHector Martin // SPDX-License-Identifier: GPL-2.0-only
26286bbb4SHector Martin /*
36286bbb4SHector Martin * Apple SoC CPU cluster performance state driver
46286bbb4SHector Martin *
56286bbb4SHector Martin * Copyright The Asahi Linux Contributors
66286bbb4SHector Martin *
76286bbb4SHector Martin * Based on scpi-cpufreq.c
86286bbb4SHector Martin */
96286bbb4SHector Martin
106286bbb4SHector Martin #include <linux/bitfield.h>
116286bbb4SHector Martin #include <linux/bitops.h>
126286bbb4SHector Martin #include <linux/cpu.h>
136286bbb4SHector Martin #include <linux/cpufreq.h>
146286bbb4SHector Martin #include <linux/cpumask.h>
156286bbb4SHector Martin #include <linux/delay.h>
166286bbb4SHector Martin #include <linux/err.h>
176286bbb4SHector Martin #include <linux/io.h>
186286bbb4SHector Martin #include <linux/iopoll.h>
196286bbb4SHector Martin #include <linux/module.h>
206286bbb4SHector Martin #include <linux/of.h>
216286bbb4SHector Martin #include <linux/of_address.h>
226286bbb4SHector Martin #include <linux/pm_opp.h>
236286bbb4SHector Martin #include <linux/slab.h>
246286bbb4SHector Martin
256286bbb4SHector Martin #define APPLE_DVFS_CMD 0x20
266286bbb4SHector Martin #define APPLE_DVFS_CMD_BUSY BIT(31)
276286bbb4SHector Martin #define APPLE_DVFS_CMD_SET BIT(25)
286286bbb4SHector Martin #define APPLE_DVFS_CMD_PS2 GENMASK(16, 12)
296286bbb4SHector Martin #define APPLE_DVFS_CMD_PS1 GENMASK(4, 0)
306286bbb4SHector Martin
316286bbb4SHector Martin /* Same timebase as CPU counter (24MHz) */
326286bbb4SHector Martin #define APPLE_DVFS_LAST_CHG_TIME 0x38
336286bbb4SHector Martin
346286bbb4SHector Martin /*
356286bbb4SHector Martin * Apple ran out of bits and had to shift this in T8112...
366286bbb4SHector Martin */
376286bbb4SHector Martin #define APPLE_DVFS_STATUS 0x50
386286bbb4SHector Martin #define APPLE_DVFS_STATUS_CUR_PS_T8103 GENMASK(7, 4)
396286bbb4SHector Martin #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103 4
406286bbb4SHector Martin #define APPLE_DVFS_STATUS_TGT_PS_T8103 GENMASK(3, 0)
416286bbb4SHector Martin #define APPLE_DVFS_STATUS_CUR_PS_T8112 GENMASK(9, 5)
426286bbb4SHector Martin #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112 5
436286bbb4SHector Martin #define APPLE_DVFS_STATUS_TGT_PS_T8112 GENMASK(4, 0)
446286bbb4SHector Martin
456286bbb4SHector Martin /*
466286bbb4SHector Martin * Div is +1, base clock is 12MHz on existing SoCs.
476286bbb4SHector Martin * For documentation purposes. We use the OPP table to
486286bbb4SHector Martin * get the frequency.
496286bbb4SHector Martin */
506286bbb4SHector Martin #define APPLE_DVFS_PLL_STATUS 0xc0
516286bbb4SHector Martin #define APPLE_DVFS_PLL_FACTOR 0xc8
526286bbb4SHector Martin #define APPLE_DVFS_PLL_FACTOR_MULT GENMASK(31, 16)
536286bbb4SHector Martin #define APPLE_DVFS_PLL_FACTOR_DIV GENMASK(15, 0)
546286bbb4SHector Martin
556286bbb4SHector Martin #define APPLE_DVFS_TRANSITION_TIMEOUT 100
566286bbb4SHector Martin
576286bbb4SHector Martin struct apple_soc_cpufreq_info {
586286bbb4SHector Martin u64 max_pstate;
596286bbb4SHector Martin u64 cur_pstate_mask;
606286bbb4SHector Martin u64 cur_pstate_shift;
616286bbb4SHector Martin };
626286bbb4SHector Martin
636286bbb4SHector Martin struct apple_cpu_priv {
646286bbb4SHector Martin struct device *cpu_dev;
656286bbb4SHector Martin void __iomem *reg_base;
666286bbb4SHector Martin const struct apple_soc_cpufreq_info *info;
676286bbb4SHector Martin };
686286bbb4SHector Martin
696286bbb4SHector Martin static struct cpufreq_driver apple_soc_cpufreq_driver;
706286bbb4SHector Martin
716286bbb4SHector Martin static const struct apple_soc_cpufreq_info soc_t8103_info = {
726286bbb4SHector Martin .max_pstate = 15,
736286bbb4SHector Martin .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103,
746286bbb4SHector Martin .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103,
756286bbb4SHector Martin };
766286bbb4SHector Martin
776286bbb4SHector Martin static const struct apple_soc_cpufreq_info soc_t8112_info = {
786286bbb4SHector Martin .max_pstate = 31,
796286bbb4SHector Martin .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112,
806286bbb4SHector Martin .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112,
816286bbb4SHector Martin };
826286bbb4SHector Martin
836286bbb4SHector Martin static const struct apple_soc_cpufreq_info soc_default_info = {
846286bbb4SHector Martin .max_pstate = 15,
856286bbb4SHector Martin .cur_pstate_mask = 0, /* fallback */
866286bbb4SHector Martin };
876286bbb4SHector Martin
886286bbb4SHector Martin static const struct of_device_id apple_soc_cpufreq_of_match[] = {
896286bbb4SHector Martin {
906286bbb4SHector Martin .compatible = "apple,t8103-cluster-cpufreq",
916286bbb4SHector Martin .data = &soc_t8103_info,
926286bbb4SHector Martin },
936286bbb4SHector Martin {
946286bbb4SHector Martin .compatible = "apple,t8112-cluster-cpufreq",
956286bbb4SHector Martin .data = &soc_t8112_info,
966286bbb4SHector Martin },
976286bbb4SHector Martin {
986286bbb4SHector Martin .compatible = "apple,cluster-cpufreq",
996286bbb4SHector Martin .data = &soc_default_info,
1006286bbb4SHector Martin },
1016286bbb4SHector Martin {}
1026286bbb4SHector Martin };
1036286bbb4SHector Martin
apple_soc_cpufreq_get_rate(unsigned int cpu)1046286bbb4SHector Martin static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu)
1056286bbb4SHector Martin {
1066286bbb4SHector Martin struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
1076286bbb4SHector Martin struct apple_cpu_priv *priv = policy->driver_data;
1086286bbb4SHector Martin struct cpufreq_frequency_table *p;
1096286bbb4SHector Martin unsigned int pstate;
1106286bbb4SHector Martin
1116286bbb4SHector Martin if (priv->info->cur_pstate_mask) {
1126286bbb4SHector Martin u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_STATUS);
1136286bbb4SHector Martin
1146286bbb4SHector Martin pstate = (reg & priv->info->cur_pstate_mask) >> priv->info->cur_pstate_shift;
1156286bbb4SHector Martin } else {
1166286bbb4SHector Martin /*
1176286bbb4SHector Martin * For the fallback case we might not know the layout of DVFS_STATUS,
1186286bbb4SHector Martin * so just use the command register value (which ignores boost limitations).
1196286bbb4SHector Martin */
1206286bbb4SHector Martin u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD);
1216286bbb4SHector Martin
1226286bbb4SHector Martin pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg);
1236286bbb4SHector Martin }
1246286bbb4SHector Martin
1256286bbb4SHector Martin cpufreq_for_each_valid_entry(p, policy->freq_table)
1266286bbb4SHector Martin if (p->driver_data == pstate)
1276286bbb4SHector Martin return p->frequency;
1286286bbb4SHector Martin
1296286bbb4SHector Martin dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n",
1306286bbb4SHector Martin pstate);
1316286bbb4SHector Martin return 0;
1326286bbb4SHector Martin }
1336286bbb4SHector Martin
apple_soc_cpufreq_set_target(struct cpufreq_policy * policy,unsigned int index)1346286bbb4SHector Martin static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy,
1356286bbb4SHector Martin unsigned int index)
1366286bbb4SHector Martin {
1376286bbb4SHector Martin struct apple_cpu_priv *priv = policy->driver_data;
1386286bbb4SHector Martin unsigned int pstate = policy->freq_table[index].driver_data;
1396286bbb4SHector Martin u64 reg;
1406286bbb4SHector Martin
1416286bbb4SHector Martin /* Fallback for newer SoCs */
1426286bbb4SHector Martin if (index > priv->info->max_pstate)
1436286bbb4SHector Martin index = priv->info->max_pstate;
1446286bbb4SHector Martin
1456286bbb4SHector Martin if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg,
1466286bbb4SHector Martin !(reg & APPLE_DVFS_CMD_BUSY), 2,
1476286bbb4SHector Martin APPLE_DVFS_TRANSITION_TIMEOUT)) {
1486286bbb4SHector Martin return -EIO;
1496286bbb4SHector Martin }
1506286bbb4SHector Martin
1516286bbb4SHector Martin reg &= ~(APPLE_DVFS_CMD_PS1 | APPLE_DVFS_CMD_PS2);
1526286bbb4SHector Martin reg |= FIELD_PREP(APPLE_DVFS_CMD_PS1, pstate);
1536286bbb4SHector Martin reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate);
1546286bbb4SHector Martin reg |= APPLE_DVFS_CMD_SET;
1556286bbb4SHector Martin
1566286bbb4SHector Martin writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD);
1576286bbb4SHector Martin
1586286bbb4SHector Martin return 0;
1596286bbb4SHector Martin }
1606286bbb4SHector Martin
apple_soc_cpufreq_fast_switch(struct cpufreq_policy * policy,unsigned int target_freq)1616286bbb4SHector Martin static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy,
1626286bbb4SHector Martin unsigned int target_freq)
1636286bbb4SHector Martin {
1646286bbb4SHector Martin if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0)
1656286bbb4SHector Martin return 0;
1666286bbb4SHector Martin
1676286bbb4SHector Martin return policy->freq_table[policy->cached_resolved_idx].frequency;
1686286bbb4SHector Martin }
1696286bbb4SHector Martin
apple_soc_cpufreq_find_cluster(struct cpufreq_policy * policy,void __iomem ** reg_base,const struct apple_soc_cpufreq_info ** info)1706286bbb4SHector Martin static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy,
1716286bbb4SHector Martin void __iomem **reg_base,
1726286bbb4SHector Martin const struct apple_soc_cpufreq_info **info)
1736286bbb4SHector Martin {
1746286bbb4SHector Martin struct of_phandle_args args;
1756286bbb4SHector Martin const struct of_device_id *match;
1766286bbb4SHector Martin int ret = 0;
1776286bbb4SHector Martin
1786286bbb4SHector Martin ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains",
1796286bbb4SHector Martin "#performance-domain-cells",
1806286bbb4SHector Martin policy->cpus, &args);
1816286bbb4SHector Martin if (ret < 0)
1826286bbb4SHector Martin return ret;
1836286bbb4SHector Martin
1846286bbb4SHector Martin match = of_match_node(apple_soc_cpufreq_of_match, args.np);
1856286bbb4SHector Martin of_node_put(args.np);
1866286bbb4SHector Martin if (!match)
1876286bbb4SHector Martin return -ENODEV;
1886286bbb4SHector Martin
1896286bbb4SHector Martin *info = match->data;
1906286bbb4SHector Martin
1916286bbb4SHector Martin *reg_base = of_iomap(args.np, 0);
192*f4352362SDan Carpenter if (!*reg_base)
193*f4352362SDan Carpenter return -ENOMEM;
1946286bbb4SHector Martin
1956286bbb4SHector Martin return 0;
1966286bbb4SHector Martin }
1976286bbb4SHector Martin
1986286bbb4SHector Martin static struct freq_attr *apple_soc_cpufreq_hw_attr[] = {
1996286bbb4SHector Martin &cpufreq_freq_attr_scaling_available_freqs,
2006286bbb4SHector Martin NULL, /* Filled in below if boost is enabled */
2016286bbb4SHector Martin NULL,
2026286bbb4SHector Martin };
2036286bbb4SHector Martin
apple_soc_cpufreq_init(struct cpufreq_policy * policy)2046286bbb4SHector Martin static int apple_soc_cpufreq_init(struct cpufreq_policy *policy)
2056286bbb4SHector Martin {
2066286bbb4SHector Martin int ret, i;
2076286bbb4SHector Martin unsigned int transition_latency;
2086286bbb4SHector Martin void __iomem *reg_base;
2096286bbb4SHector Martin struct device *cpu_dev;
2106286bbb4SHector Martin struct apple_cpu_priv *priv;
2116286bbb4SHector Martin const struct apple_soc_cpufreq_info *info;
2126286bbb4SHector Martin struct cpufreq_frequency_table *freq_table;
2136286bbb4SHector Martin
2146286bbb4SHector Martin cpu_dev = get_cpu_device(policy->cpu);
2156286bbb4SHector Martin if (!cpu_dev) {
2166286bbb4SHector Martin pr_err("failed to get cpu%d device\n", policy->cpu);
2176286bbb4SHector Martin return -ENODEV;
2186286bbb4SHector Martin }
2196286bbb4SHector Martin
2206286bbb4SHector Martin ret = dev_pm_opp_of_add_table(cpu_dev);
2216286bbb4SHector Martin if (ret < 0) {
2226286bbb4SHector Martin dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret);
2236286bbb4SHector Martin return ret;
2246286bbb4SHector Martin }
2256286bbb4SHector Martin
2266286bbb4SHector Martin ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info);
2276286bbb4SHector Martin if (ret) {
2286286bbb4SHector Martin dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret);
2296286bbb4SHector Martin return ret;
2306286bbb4SHector Martin }
2316286bbb4SHector Martin
2326286bbb4SHector Martin ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
2336286bbb4SHector Martin if (ret) {
2346286bbb4SHector Martin dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret);
2356286bbb4SHector Martin goto out_iounmap;
2366286bbb4SHector Martin }
2376286bbb4SHector Martin
2386286bbb4SHector Martin ret = dev_pm_opp_get_opp_count(cpu_dev);
2396286bbb4SHector Martin if (ret <= 0) {
2406286bbb4SHector Martin dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
2416286bbb4SHector Martin ret = -EPROBE_DEFER;
2426286bbb4SHector Martin goto out_free_opp;
2436286bbb4SHector Martin }
2446286bbb4SHector Martin
2456286bbb4SHector Martin priv = kzalloc(sizeof(*priv), GFP_KERNEL);
2466286bbb4SHector Martin if (!priv) {
2476286bbb4SHector Martin ret = -ENOMEM;
2486286bbb4SHector Martin goto out_free_opp;
2496286bbb4SHector Martin }
2506286bbb4SHector Martin
2516286bbb4SHector Martin ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
2526286bbb4SHector Martin if (ret) {
2536286bbb4SHector Martin dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
2546286bbb4SHector Martin goto out_free_priv;
2556286bbb4SHector Martin }
2566286bbb4SHector Martin
2576286bbb4SHector Martin /* Get OPP levels (p-state indexes) and stash them in driver_data */
2586286bbb4SHector Martin for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) {
2596286bbb4SHector Martin unsigned long rate = freq_table[i].frequency * 1000 + 999;
2606286bbb4SHector Martin struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate);
2616286bbb4SHector Martin
2626286bbb4SHector Martin if (IS_ERR(opp)) {
2636286bbb4SHector Martin ret = PTR_ERR(opp);
2646286bbb4SHector Martin goto out_free_cpufreq_table;
2656286bbb4SHector Martin }
2666286bbb4SHector Martin freq_table[i].driver_data = dev_pm_opp_get_level(opp);
2676286bbb4SHector Martin dev_pm_opp_put(opp);
2686286bbb4SHector Martin }
2696286bbb4SHector Martin
2706286bbb4SHector Martin priv->cpu_dev = cpu_dev;
2716286bbb4SHector Martin priv->reg_base = reg_base;
2726286bbb4SHector Martin priv->info = info;
2736286bbb4SHector Martin policy->driver_data = priv;
2746286bbb4SHector Martin policy->freq_table = freq_table;
2756286bbb4SHector Martin
2766286bbb4SHector Martin transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev);
2776286bbb4SHector Martin if (!transition_latency)
2786286bbb4SHector Martin transition_latency = CPUFREQ_ETERNAL;
2796286bbb4SHector Martin
2806286bbb4SHector Martin policy->cpuinfo.transition_latency = transition_latency;
2816286bbb4SHector Martin policy->dvfs_possible_from_any_cpu = true;
2826286bbb4SHector Martin policy->fast_switch_possible = true;
283c9565417SHector Martin policy->suspend_freq = freq_table[0].frequency;
2846286bbb4SHector Martin
2856286bbb4SHector Martin if (policy_has_boost_freq(policy)) {
2866286bbb4SHector Martin ret = cpufreq_enable_boost_support();
2876286bbb4SHector Martin if (ret) {
2886286bbb4SHector Martin dev_warn(cpu_dev, "failed to enable boost: %d\n", ret);
2896286bbb4SHector Martin } else {
2906286bbb4SHector Martin apple_soc_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs;
2916286bbb4SHector Martin apple_soc_cpufreq_driver.boost_enabled = true;
2926286bbb4SHector Martin }
2936286bbb4SHector Martin }
2946286bbb4SHector Martin
2956286bbb4SHector Martin return 0;
2966286bbb4SHector Martin
2976286bbb4SHector Martin out_free_cpufreq_table:
2986286bbb4SHector Martin dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
2996286bbb4SHector Martin out_free_priv:
3006286bbb4SHector Martin kfree(priv);
3016286bbb4SHector Martin out_free_opp:
3026286bbb4SHector Martin dev_pm_opp_remove_all_dynamic(cpu_dev);
3036286bbb4SHector Martin out_iounmap:
3046286bbb4SHector Martin iounmap(reg_base);
3056286bbb4SHector Martin return ret;
3066286bbb4SHector Martin }
3076286bbb4SHector Martin
apple_soc_cpufreq_exit(struct cpufreq_policy * policy)3086286bbb4SHector Martin static int apple_soc_cpufreq_exit(struct cpufreq_policy *policy)
3096286bbb4SHector Martin {
3106286bbb4SHector Martin struct apple_cpu_priv *priv = policy->driver_data;
3116286bbb4SHector Martin
3126286bbb4SHector Martin dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
3136286bbb4SHector Martin dev_pm_opp_remove_all_dynamic(priv->cpu_dev);
3146286bbb4SHector Martin iounmap(priv->reg_base);
3156286bbb4SHector Martin kfree(priv);
3166286bbb4SHector Martin
3176286bbb4SHector Martin return 0;
3186286bbb4SHector Martin }
3196286bbb4SHector Martin
3206286bbb4SHector Martin static struct cpufreq_driver apple_soc_cpufreq_driver = {
3216286bbb4SHector Martin .name = "apple-cpufreq",
3226286bbb4SHector Martin .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
3236286bbb4SHector Martin CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV,
3246286bbb4SHector Martin .verify = cpufreq_generic_frequency_table_verify,
3256286bbb4SHector Martin .get = apple_soc_cpufreq_get_rate,
3266286bbb4SHector Martin .init = apple_soc_cpufreq_init,
3276286bbb4SHector Martin .exit = apple_soc_cpufreq_exit,
3286286bbb4SHector Martin .target_index = apple_soc_cpufreq_set_target,
3296286bbb4SHector Martin .fast_switch = apple_soc_cpufreq_fast_switch,
3306286bbb4SHector Martin .register_em = cpufreq_register_em_with_opp,
3316286bbb4SHector Martin .attr = apple_soc_cpufreq_hw_attr,
332c9565417SHector Martin .suspend = cpufreq_generic_suspend,
3336286bbb4SHector Martin };
3346286bbb4SHector Martin
apple_soc_cpufreq_module_init(void)3356286bbb4SHector Martin static int __init apple_soc_cpufreq_module_init(void)
3366286bbb4SHector Martin {
3376286bbb4SHector Martin if (!of_machine_is_compatible("apple,arm-platform"))
3386286bbb4SHector Martin return -ENODEV;
3396286bbb4SHector Martin
3406286bbb4SHector Martin return cpufreq_register_driver(&apple_soc_cpufreq_driver);
3416286bbb4SHector Martin }
3426286bbb4SHector Martin module_init(apple_soc_cpufreq_module_init);
3436286bbb4SHector Martin
apple_soc_cpufreq_module_exit(void)3446286bbb4SHector Martin static void __exit apple_soc_cpufreq_module_exit(void)
3456286bbb4SHector Martin {
3466286bbb4SHector Martin cpufreq_unregister_driver(&apple_soc_cpufreq_driver);
3476286bbb4SHector Martin }
3486286bbb4SHector Martin module_exit(apple_soc_cpufreq_module_exit);
3496286bbb4SHector Martin
3506286bbb4SHector Martin MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match);
3516286bbb4SHector Martin MODULE_AUTHOR("Hector Martin <marcan@marcan.st>");
3526286bbb4SHector Martin MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver");
3536286bbb4SHector Martin MODULE_LICENSE("GPL");
354