1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Apple SoC CPU cluster performance state driver
4 *
5 * Copyright The Asahi Linux Contributors
6 *
7 * Based on scpi-cpufreq.c
8 */
9
10 #include <linux/bitfield.h>
11 #include <linux/bitops.h>
12 #include <linux/cpu.h>
13 #include <linux/cpufreq.h>
14 #include <linux/cpumask.h>
15 #include <linux/delay.h>
16 #include <linux/err.h>
17 #include <linux/io.h>
18 #include <linux/iopoll.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/of_address.h>
22 #include <linux/pm_opp.h>
23 #include <linux/slab.h>
24
25 #define APPLE_DVFS_CMD 0x20
26 #define APPLE_DVFS_CMD_BUSY BIT(31)
27 #define APPLE_DVFS_CMD_SET BIT(25)
28 #define APPLE_DVFS_CMD_PS2 GENMASK(16, 12)
29 #define APPLE_DVFS_CMD_PS1 GENMASK(4, 0)
30
31 /* Same timebase as CPU counter (24MHz) */
32 #define APPLE_DVFS_LAST_CHG_TIME 0x38
33
34 /*
35 * Apple ran out of bits and had to shift this in T8112...
36 */
37 #define APPLE_DVFS_STATUS 0x50
38 #define APPLE_DVFS_STATUS_CUR_PS_T8103 GENMASK(7, 4)
39 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103 4
40 #define APPLE_DVFS_STATUS_TGT_PS_T8103 GENMASK(3, 0)
41 #define APPLE_DVFS_STATUS_CUR_PS_T8112 GENMASK(9, 5)
42 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112 5
43 #define APPLE_DVFS_STATUS_TGT_PS_T8112 GENMASK(4, 0)
44
45 /*
46 * Div is +1, base clock is 12MHz on existing SoCs.
47 * For documentation purposes. We use the OPP table to
48 * get the frequency.
49 */
50 #define APPLE_DVFS_PLL_STATUS 0xc0
51 #define APPLE_DVFS_PLL_FACTOR 0xc8
52 #define APPLE_DVFS_PLL_FACTOR_MULT GENMASK(31, 16)
53 #define APPLE_DVFS_PLL_FACTOR_DIV GENMASK(15, 0)
54
55 #define APPLE_DVFS_TRANSITION_TIMEOUT 100
56
57 struct apple_soc_cpufreq_info {
58 u64 max_pstate;
59 u64 cur_pstate_mask;
60 u64 cur_pstate_shift;
61 };
62
63 struct apple_cpu_priv {
64 struct device *cpu_dev;
65 void __iomem *reg_base;
66 const struct apple_soc_cpufreq_info *info;
67 };
68
69 static struct cpufreq_driver apple_soc_cpufreq_driver;
70
71 static const struct apple_soc_cpufreq_info soc_t8103_info = {
72 .max_pstate = 15,
73 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103,
74 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103,
75 };
76
77 static const struct apple_soc_cpufreq_info soc_t8112_info = {
78 .max_pstate = 31,
79 .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112,
80 .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112,
81 };
82
83 static const struct apple_soc_cpufreq_info soc_default_info = {
84 .max_pstate = 15,
85 .cur_pstate_mask = 0, /* fallback */
86 };
87
88 static const struct of_device_id apple_soc_cpufreq_of_match[] = {
89 {
90 .compatible = "apple,t8103-cluster-cpufreq",
91 .data = &soc_t8103_info,
92 },
93 {
94 .compatible = "apple,t8112-cluster-cpufreq",
95 .data = &soc_t8112_info,
96 },
97 {
98 .compatible = "apple,cluster-cpufreq",
99 .data = &soc_default_info,
100 },
101 {}
102 };
103
apple_soc_cpufreq_get_rate(unsigned int cpu)104 static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu)
105 {
106 struct cpufreq_policy *policy;
107 struct apple_cpu_priv *priv;
108 struct cpufreq_frequency_table *p;
109 unsigned int pstate;
110
111 policy = cpufreq_cpu_get_raw(cpu);
112 if (unlikely(!policy))
113 return 0;
114
115 priv = policy->driver_data;
116
117 if (priv->info->cur_pstate_mask) {
118 u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_STATUS);
119
120 pstate = (reg & priv->info->cur_pstate_mask) >> priv->info->cur_pstate_shift;
121 } else {
122 /*
123 * For the fallback case we might not know the layout of DVFS_STATUS,
124 * so just use the command register value (which ignores boost limitations).
125 */
126 u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD);
127
128 pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg);
129 }
130
131 cpufreq_for_each_valid_entry(p, policy->freq_table)
132 if (p->driver_data == pstate)
133 return p->frequency;
134
135 dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n",
136 pstate);
137 return 0;
138 }
139
apple_soc_cpufreq_set_target(struct cpufreq_policy * policy,unsigned int index)140 static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy,
141 unsigned int index)
142 {
143 struct apple_cpu_priv *priv = policy->driver_data;
144 unsigned int pstate = policy->freq_table[index].driver_data;
145 u64 reg;
146
147 /* Fallback for newer SoCs */
148 if (index > priv->info->max_pstate)
149 index = priv->info->max_pstate;
150
151 if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg,
152 !(reg & APPLE_DVFS_CMD_BUSY), 2,
153 APPLE_DVFS_TRANSITION_TIMEOUT)) {
154 return -EIO;
155 }
156
157 reg &= ~(APPLE_DVFS_CMD_PS1 | APPLE_DVFS_CMD_PS2);
158 reg |= FIELD_PREP(APPLE_DVFS_CMD_PS1, pstate);
159 reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate);
160 reg |= APPLE_DVFS_CMD_SET;
161
162 writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD);
163
164 return 0;
165 }
166
apple_soc_cpufreq_fast_switch(struct cpufreq_policy * policy,unsigned int target_freq)167 static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy,
168 unsigned int target_freq)
169 {
170 if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0)
171 return 0;
172
173 return policy->freq_table[policy->cached_resolved_idx].frequency;
174 }
175
apple_soc_cpufreq_find_cluster(struct cpufreq_policy * policy,void __iomem ** reg_base,const struct apple_soc_cpufreq_info ** info)176 static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy,
177 void __iomem **reg_base,
178 const struct apple_soc_cpufreq_info **info)
179 {
180 struct of_phandle_args args;
181 const struct of_device_id *match;
182 int ret = 0;
183
184 ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains",
185 "#performance-domain-cells",
186 policy->cpus, &args);
187 if (ret < 0)
188 return ret;
189
190 match = of_match_node(apple_soc_cpufreq_of_match, args.np);
191 of_node_put(args.np);
192 if (!match)
193 return -ENODEV;
194
195 *info = match->data;
196
197 *reg_base = of_iomap(args.np, 0);
198 if (!*reg_base)
199 return -ENOMEM;
200
201 return 0;
202 }
203
204 static struct freq_attr *apple_soc_cpufreq_hw_attr[] = {
205 &cpufreq_freq_attr_scaling_available_freqs,
206 NULL, /* Filled in below if boost is enabled */
207 NULL,
208 };
209
apple_soc_cpufreq_init(struct cpufreq_policy * policy)210 static int apple_soc_cpufreq_init(struct cpufreq_policy *policy)
211 {
212 int ret, i;
213 unsigned int transition_latency;
214 void __iomem *reg_base;
215 struct device *cpu_dev;
216 struct apple_cpu_priv *priv;
217 const struct apple_soc_cpufreq_info *info;
218 struct cpufreq_frequency_table *freq_table;
219
220 cpu_dev = get_cpu_device(policy->cpu);
221 if (!cpu_dev) {
222 pr_err("failed to get cpu%d device\n", policy->cpu);
223 return -ENODEV;
224 }
225
226 ret = dev_pm_opp_of_add_table(cpu_dev);
227 if (ret < 0) {
228 dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret);
229 return ret;
230 }
231
232 ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info);
233 if (ret) {
234 dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret);
235 return ret;
236 }
237
238 ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
239 if (ret) {
240 dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret);
241 goto out_iounmap;
242 }
243
244 ret = dev_pm_opp_get_opp_count(cpu_dev);
245 if (ret <= 0) {
246 dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
247 ret = -EPROBE_DEFER;
248 goto out_free_opp;
249 }
250
251 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
252 if (!priv) {
253 ret = -ENOMEM;
254 goto out_free_opp;
255 }
256
257 ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
258 if (ret) {
259 dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
260 goto out_free_priv;
261 }
262
263 /* Get OPP levels (p-state indexes) and stash them in driver_data */
264 for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) {
265 unsigned long rate = freq_table[i].frequency * 1000 + 999;
266 struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate);
267
268 if (IS_ERR(opp)) {
269 ret = PTR_ERR(opp);
270 goto out_free_cpufreq_table;
271 }
272 freq_table[i].driver_data = dev_pm_opp_get_level(opp);
273 dev_pm_opp_put(opp);
274 }
275
276 priv->cpu_dev = cpu_dev;
277 priv->reg_base = reg_base;
278 priv->info = info;
279 policy->driver_data = priv;
280 policy->freq_table = freq_table;
281
282 transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev);
283 if (!transition_latency)
284 transition_latency = CPUFREQ_ETERNAL;
285
286 policy->cpuinfo.transition_latency = transition_latency;
287 policy->dvfs_possible_from_any_cpu = true;
288 policy->fast_switch_possible = true;
289 policy->suspend_freq = freq_table[0].frequency;
290
291 if (policy_has_boost_freq(policy)) {
292 ret = cpufreq_enable_boost_support();
293 if (ret) {
294 dev_warn(cpu_dev, "failed to enable boost: %d\n", ret);
295 } else {
296 apple_soc_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs;
297 apple_soc_cpufreq_driver.boost_enabled = true;
298 }
299 }
300
301 return 0;
302
303 out_free_cpufreq_table:
304 dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
305 out_free_priv:
306 kfree(priv);
307 out_free_opp:
308 dev_pm_opp_remove_all_dynamic(cpu_dev);
309 out_iounmap:
310 iounmap(reg_base);
311 return ret;
312 }
313
apple_soc_cpufreq_exit(struct cpufreq_policy * policy)314 static int apple_soc_cpufreq_exit(struct cpufreq_policy *policy)
315 {
316 struct apple_cpu_priv *priv = policy->driver_data;
317
318 dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
319 dev_pm_opp_remove_all_dynamic(priv->cpu_dev);
320 iounmap(priv->reg_base);
321 kfree(priv);
322
323 return 0;
324 }
325
326 static struct cpufreq_driver apple_soc_cpufreq_driver = {
327 .name = "apple-cpufreq",
328 .flags = CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
329 CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV,
330 .verify = cpufreq_generic_frequency_table_verify,
331 .get = apple_soc_cpufreq_get_rate,
332 .init = apple_soc_cpufreq_init,
333 .exit = apple_soc_cpufreq_exit,
334 .target_index = apple_soc_cpufreq_set_target,
335 .fast_switch = apple_soc_cpufreq_fast_switch,
336 .register_em = cpufreq_register_em_with_opp,
337 .attr = apple_soc_cpufreq_hw_attr,
338 .suspend = cpufreq_generic_suspend,
339 };
340
apple_soc_cpufreq_module_init(void)341 static int __init apple_soc_cpufreq_module_init(void)
342 {
343 if (!of_machine_is_compatible("apple,arm-platform"))
344 return -ENODEV;
345
346 return cpufreq_register_driver(&apple_soc_cpufreq_driver);
347 }
348 module_init(apple_soc_cpufreq_module_init);
349
apple_soc_cpufreq_module_exit(void)350 static void __exit apple_soc_cpufreq_module_exit(void)
351 {
352 cpufreq_unregister_driver(&apple_soc_cpufreq_driver);
353 }
354 module_exit(apple_soc_cpufreq_module_exit);
355
356 MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match);
357 MODULE_AUTHOR("Hector Martin <marcan@marcan.st>");
358 MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver");
359 MODULE_LICENSE("GPL");
360