1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Apple SoC CPU cluster performance state driver
4  *
5  * Copyright The Asahi Linux Contributors
6  *
7  * Based on scpi-cpufreq.c
8  */
9 
10 #include <linux/bitfield.h>
11 #include <linux/bitops.h>
12 #include <linux/cpu.h>
13 #include <linux/cpufreq.h>
14 #include <linux/cpumask.h>
15 #include <linux/delay.h>
16 #include <linux/err.h>
17 #include <linux/io.h>
18 #include <linux/iopoll.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/of_address.h>
22 #include <linux/pm_opp.h>
23 #include <linux/slab.h>
24 
25 #define APPLE_DVFS_CMD			0x20
26 #define APPLE_DVFS_CMD_BUSY		BIT(31)
27 #define APPLE_DVFS_CMD_SET		BIT(25)
28 #define APPLE_DVFS_CMD_PS2		GENMASK(16, 12)
29 #define APPLE_DVFS_CMD_PS1		GENMASK(4, 0)
30 
31 /* Same timebase as CPU counter (24MHz) */
32 #define APPLE_DVFS_LAST_CHG_TIME	0x38
33 
34 /*
35  * Apple ran out of bits and had to shift this in T8112...
36  */
37 #define APPLE_DVFS_STATUS			0x50
38 #define APPLE_DVFS_STATUS_CUR_PS_T8103		GENMASK(7, 4)
39 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103	4
40 #define APPLE_DVFS_STATUS_TGT_PS_T8103		GENMASK(3, 0)
41 #define APPLE_DVFS_STATUS_CUR_PS_T8112		GENMASK(9, 5)
42 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112	5
43 #define APPLE_DVFS_STATUS_TGT_PS_T8112		GENMASK(4, 0)
44 
45 /*
46  * Div is +1, base clock is 12MHz on existing SoCs.
47  * For documentation purposes. We use the OPP table to
48  * get the frequency.
49  */
50 #define APPLE_DVFS_PLL_STATUS		0xc0
51 #define APPLE_DVFS_PLL_FACTOR		0xc8
52 #define APPLE_DVFS_PLL_FACTOR_MULT	GENMASK(31, 16)
53 #define APPLE_DVFS_PLL_FACTOR_DIV	GENMASK(15, 0)
54 
55 #define APPLE_DVFS_TRANSITION_TIMEOUT 100
56 
57 struct apple_soc_cpufreq_info {
58 	u64 max_pstate;
59 	u64 cur_pstate_mask;
60 	u64 cur_pstate_shift;
61 };
62 
63 struct apple_cpu_priv {
64 	struct device *cpu_dev;
65 	void __iomem *reg_base;
66 	const struct apple_soc_cpufreq_info *info;
67 };
68 
69 static struct cpufreq_driver apple_soc_cpufreq_driver;
70 
71 static const struct apple_soc_cpufreq_info soc_t8103_info = {
72 	.max_pstate = 15,
73 	.cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103,
74 	.cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103,
75 };
76 
77 static const struct apple_soc_cpufreq_info soc_t8112_info = {
78 	.max_pstate = 31,
79 	.cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112,
80 	.cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112,
81 };
82 
83 static const struct apple_soc_cpufreq_info soc_default_info = {
84 	.max_pstate = 15,
85 	.cur_pstate_mask = 0, /* fallback */
86 };
87 
88 static const struct of_device_id apple_soc_cpufreq_of_match[] = {
89 	{
90 		.compatible = "apple,t8103-cluster-cpufreq",
91 		.data = &soc_t8103_info,
92 	},
93 	{
94 		.compatible = "apple,t8112-cluster-cpufreq",
95 		.data = &soc_t8112_info,
96 	},
97 	{
98 		.compatible = "apple,cluster-cpufreq",
99 		.data = &soc_default_info,
100 	},
101 	{}
102 };
103 
104 static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu)
105 {
106 	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
107 	struct apple_cpu_priv *priv = policy->driver_data;
108 	struct cpufreq_frequency_table *p;
109 	unsigned int pstate;
110 
111 	if (priv->info->cur_pstate_mask) {
112 		u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_STATUS);
113 
114 		pstate = (reg & priv->info->cur_pstate_mask) >>  priv->info->cur_pstate_shift;
115 	} else {
116 		/*
117 		 * For the fallback case we might not know the layout of DVFS_STATUS,
118 		 * so just use the command register value (which ignores boost limitations).
119 		 */
120 		u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD);
121 
122 		pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg);
123 	}
124 
125 	cpufreq_for_each_valid_entry(p, policy->freq_table)
126 		if (p->driver_data == pstate)
127 			return p->frequency;
128 
129 	dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n",
130 		pstate);
131 	return 0;
132 }
133 
134 static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy,
135 					unsigned int index)
136 {
137 	struct apple_cpu_priv *priv = policy->driver_data;
138 	unsigned int pstate = policy->freq_table[index].driver_data;
139 	u64 reg;
140 
141 	/* Fallback for newer SoCs */
142 	if (index > priv->info->max_pstate)
143 		index = priv->info->max_pstate;
144 
145 	if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg,
146 				      !(reg & APPLE_DVFS_CMD_BUSY), 2,
147 				      APPLE_DVFS_TRANSITION_TIMEOUT)) {
148 		return -EIO;
149 	}
150 
151 	reg &= ~(APPLE_DVFS_CMD_PS1 | APPLE_DVFS_CMD_PS2);
152 	reg |= FIELD_PREP(APPLE_DVFS_CMD_PS1, pstate);
153 	reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate);
154 	reg |= APPLE_DVFS_CMD_SET;
155 
156 	writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD);
157 
158 	return 0;
159 }
160 
161 static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy,
162 						  unsigned int target_freq)
163 {
164 	if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0)
165 		return 0;
166 
167 	return policy->freq_table[policy->cached_resolved_idx].frequency;
168 }
169 
170 static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy,
171 					  void __iomem **reg_base,
172 					  const struct apple_soc_cpufreq_info **info)
173 {
174 	struct of_phandle_args args;
175 	const struct of_device_id *match;
176 	int ret = 0;
177 
178 	ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains",
179 						 "#performance-domain-cells",
180 						 policy->cpus, &args);
181 	if (ret < 0)
182 		return ret;
183 
184 	match = of_match_node(apple_soc_cpufreq_of_match, args.np);
185 	of_node_put(args.np);
186 	if (!match)
187 		return -ENODEV;
188 
189 	*info = match->data;
190 
191 	*reg_base = of_iomap(args.np, 0);
192 	if (IS_ERR(*reg_base))
193 		return PTR_ERR(*reg_base);
194 
195 	return 0;
196 }
197 
198 static struct freq_attr *apple_soc_cpufreq_hw_attr[] = {
199 	&cpufreq_freq_attr_scaling_available_freqs,
200 	NULL, /* Filled in below if boost is enabled */
201 	NULL,
202 };
203 
204 static int apple_soc_cpufreq_init(struct cpufreq_policy *policy)
205 {
206 	int ret, i;
207 	unsigned int transition_latency;
208 	void __iomem *reg_base;
209 	struct device *cpu_dev;
210 	struct apple_cpu_priv *priv;
211 	const struct apple_soc_cpufreq_info *info;
212 	struct cpufreq_frequency_table *freq_table;
213 
214 	cpu_dev = get_cpu_device(policy->cpu);
215 	if (!cpu_dev) {
216 		pr_err("failed to get cpu%d device\n", policy->cpu);
217 		return -ENODEV;
218 	}
219 
220 	ret = dev_pm_opp_of_add_table(cpu_dev);
221 	if (ret < 0) {
222 		dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret);
223 		return ret;
224 	}
225 
226 	ret = apple_soc_cpufreq_find_cluster(policy, &reg_base, &info);
227 	if (ret) {
228 		dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret);
229 		return ret;
230 	}
231 
232 	ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
233 	if (ret) {
234 		dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret);
235 		goto out_iounmap;
236 	}
237 
238 	ret = dev_pm_opp_get_opp_count(cpu_dev);
239 	if (ret <= 0) {
240 		dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
241 		ret = -EPROBE_DEFER;
242 		goto out_free_opp;
243 	}
244 
245 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
246 	if (!priv) {
247 		ret = -ENOMEM;
248 		goto out_free_opp;
249 	}
250 
251 	ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
252 	if (ret) {
253 		dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
254 		goto out_free_priv;
255 	}
256 
257 	/* Get OPP levels (p-state indexes) and stash them in driver_data */
258 	for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) {
259 		unsigned long rate = freq_table[i].frequency * 1000 + 999;
260 		struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate);
261 
262 		if (IS_ERR(opp)) {
263 			ret = PTR_ERR(opp);
264 			goto out_free_cpufreq_table;
265 		}
266 		freq_table[i].driver_data = dev_pm_opp_get_level(opp);
267 		dev_pm_opp_put(opp);
268 	}
269 
270 	priv->cpu_dev = cpu_dev;
271 	priv->reg_base = reg_base;
272 	priv->info = info;
273 	policy->driver_data = priv;
274 	policy->freq_table = freq_table;
275 
276 	transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev);
277 	if (!transition_latency)
278 		transition_latency = CPUFREQ_ETERNAL;
279 
280 	policy->cpuinfo.transition_latency = transition_latency;
281 	policy->dvfs_possible_from_any_cpu = true;
282 	policy->fast_switch_possible = true;
283 	policy->suspend_freq = freq_table[0].frequency;
284 
285 	if (policy_has_boost_freq(policy)) {
286 		ret = cpufreq_enable_boost_support();
287 		if (ret) {
288 			dev_warn(cpu_dev, "failed to enable boost: %d\n", ret);
289 		} else {
290 			apple_soc_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs;
291 			apple_soc_cpufreq_driver.boost_enabled = true;
292 		}
293 	}
294 
295 	return 0;
296 
297 out_free_cpufreq_table:
298 	dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
299 out_free_priv:
300 	kfree(priv);
301 out_free_opp:
302 	dev_pm_opp_remove_all_dynamic(cpu_dev);
303 out_iounmap:
304 	iounmap(reg_base);
305 	return ret;
306 }
307 
308 static int apple_soc_cpufreq_exit(struct cpufreq_policy *policy)
309 {
310 	struct apple_cpu_priv *priv = policy->driver_data;
311 
312 	dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
313 	dev_pm_opp_remove_all_dynamic(priv->cpu_dev);
314 	iounmap(priv->reg_base);
315 	kfree(priv);
316 
317 	return 0;
318 }
319 
320 static struct cpufreq_driver apple_soc_cpufreq_driver = {
321 	.name		= "apple-cpufreq",
322 	.flags		= CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
323 			  CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV,
324 	.verify		= cpufreq_generic_frequency_table_verify,
325 	.get		= apple_soc_cpufreq_get_rate,
326 	.init		= apple_soc_cpufreq_init,
327 	.exit		= apple_soc_cpufreq_exit,
328 	.target_index	= apple_soc_cpufreq_set_target,
329 	.fast_switch	= apple_soc_cpufreq_fast_switch,
330 	.register_em	= cpufreq_register_em_with_opp,
331 	.attr		= apple_soc_cpufreq_hw_attr,
332 	.suspend	= cpufreq_generic_suspend,
333 };
334 
335 static int __init apple_soc_cpufreq_module_init(void)
336 {
337 	if (!of_machine_is_compatible("apple,arm-platform"))
338 		return -ENODEV;
339 
340 	return cpufreq_register_driver(&apple_soc_cpufreq_driver);
341 }
342 module_init(apple_soc_cpufreq_module_init);
343 
344 static void __exit apple_soc_cpufreq_module_exit(void)
345 {
346 	cpufreq_unregister_driver(&apple_soc_cpufreq_driver);
347 }
348 module_exit(apple_soc_cpufreq_module_exit);
349 
350 MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match);
351 MODULE_AUTHOR("Hector Martin <marcan@marcan.st>");
352 MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver");
353 MODULE_LICENSE("GPL");
354