1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * CPU idle driver for Tegra CPUs
4 *
5 * Copyright (c) 2010-2013, NVIDIA Corporation.
6 * Copyright (c) 2011 Google, Inc.
7 * Author: Colin Cross <ccross@android.com>
8 * Gary King <gking@nvidia.com>
9 *
10 * Rework for 3.3 by Peter De Schrijver <pdeschrijver@nvidia.com>
11 *
12 * Tegra20/124 driver unification by Dmitry Osipenko <digetx@gmail.com>
13 */
14
15 #define pr_fmt(fmt) "tegra-cpuidle: " fmt
16
17 #include <linux/atomic.h>
18 #include <linux/cpuidle.h>
19 #include <linux/cpumask.h>
20 #include <linux/cpu_pm.h>
21 #include <linux/delay.h>
22 #include <linux/errno.h>
23 #include <linux/platform_device.h>
24 #include <linux/types.h>
25
26 #include <linux/clk/tegra.h>
27 #include <linux/firmware/trusted_foundations.h>
28
29 #include <soc/tegra/cpuidle.h>
30 #include <soc/tegra/flowctrl.h>
31 #include <soc/tegra/fuse.h>
32 #include <soc/tegra/irq.h>
33 #include <soc/tegra/pm.h>
34 #include <soc/tegra/pmc.h>
35
36 #include <asm/cpuidle.h>
37 #include <asm/firmware.h>
38 #include <asm/smp_plat.h>
39 #include <asm/suspend.h>
40
41 enum tegra_state {
42 TEGRA_C1,
43 TEGRA_C7,
44 TEGRA_CC6,
45 TEGRA_STATE_COUNT,
46 };
47
48 static atomic_t tegra_idle_barrier;
49 static atomic_t tegra_abort_flag;
50
tegra_cpuidle_report_cpus_state(void)51 static void tegra_cpuidle_report_cpus_state(void)
52 {
53 unsigned long cpu, lcpu, csr;
54
55 for_each_cpu(lcpu, cpu_possible_mask) {
56 cpu = cpu_logical_map(lcpu);
57 csr = flowctrl_read_cpu_csr(cpu);
58
59 pr_err("cpu%lu: online=%d flowctrl_csr=0x%08lx\n",
60 cpu, cpu_online(lcpu), csr);
61 }
62 }
63
tegra_cpuidle_wait_for_secondary_cpus_parking(void)64 static int tegra_cpuidle_wait_for_secondary_cpus_parking(void)
65 {
66 unsigned int retries = 3;
67
68 while (retries--) {
69 unsigned int delay_us = 10;
70 unsigned int timeout_us = 500 * 1000 / delay_us;
71
72 /*
73 * The primary CPU0 core shall wait for the secondaries
74 * shutdown in order to power-off CPU's cluster safely.
75 * The timeout value depends on the current CPU frequency,
76 * it takes about 40-150us in average and over 1000us in
77 * a worst case scenario.
78 */
79 do {
80 if (tegra_cpu_rail_off_ready())
81 return 0;
82
83 udelay(delay_us);
84
85 } while (timeout_us--);
86
87 pr_err("secondary CPU taking too long to park\n");
88
89 tegra_cpuidle_report_cpus_state();
90 }
91
92 pr_err("timed out waiting secondaries to park\n");
93
94 return -ETIMEDOUT;
95 }
96
tegra_cpuidle_unpark_secondary_cpus(void)97 static void tegra_cpuidle_unpark_secondary_cpus(void)
98 {
99 unsigned int cpu, lcpu;
100
101 for_each_cpu(lcpu, cpu_online_mask) {
102 cpu = cpu_logical_map(lcpu);
103
104 if (cpu > 0) {
105 tegra_enable_cpu_clock(cpu);
106 tegra_cpu_out_of_reset(cpu);
107 flowctrl_write_cpu_halt(cpu, 0);
108 }
109 }
110 }
111
tegra_cpuidle_cc6_enter(unsigned int cpu)112 static int tegra_cpuidle_cc6_enter(unsigned int cpu)
113 {
114 int ret;
115
116 if (cpu > 0) {
117 ret = cpu_suspend(cpu, tegra_pm_park_secondary_cpu);
118 } else {
119 ret = tegra_cpuidle_wait_for_secondary_cpus_parking();
120 if (!ret)
121 ret = tegra_pm_enter_lp2();
122
123 tegra_cpuidle_unpark_secondary_cpus();
124 }
125
126 return ret;
127 }
128
tegra_cpuidle_c7_enter(void)129 static int tegra_cpuidle_c7_enter(void)
130 {
131 int err;
132
133 err = call_firmware_op(prepare_idle, TF_PM_MODE_LP2_NOFLUSH_L2);
134 if (err && err != -ENOSYS)
135 return err;
136
137 return cpu_suspend(0, tegra30_pm_secondary_cpu_suspend);
138 }
139
tegra_cpuidle_coupled_barrier(struct cpuidle_device * dev)140 static int tegra_cpuidle_coupled_barrier(struct cpuidle_device *dev)
141 {
142 if (tegra_pending_sgi()) {
143 /*
144 * CPU got local interrupt that will be lost after GIC's
145 * shutdown because GIC driver doesn't save/restore the
146 * pending SGI state across CPU cluster PM. Abort and retry
147 * next time.
148 */
149 atomic_set(&tegra_abort_flag, 1);
150 }
151
152 cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier);
153
154 if (atomic_read(&tegra_abort_flag)) {
155 cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier);
156 atomic_set(&tegra_abort_flag, 0);
157 return -EINTR;
158 }
159
160 return 0;
161 }
162
tegra_cpuidle_state_enter(struct cpuidle_device * dev,int index,unsigned int cpu)163 static __cpuidle int tegra_cpuidle_state_enter(struct cpuidle_device *dev,
164 int index, unsigned int cpu)
165 {
166 int err;
167
168 /*
169 * CC6 state is the "CPU cluster power-off" state. In order to
170 * enter this state, at first the secondary CPU cores need to be
171 * parked into offline mode, then the last CPU should clean out
172 * remaining dirty cache lines into DRAM and trigger Flow Controller
173 * logic that turns off the cluster's power domain (which includes
174 * CPU cores, GIC and L2 cache).
175 */
176 if (index == TEGRA_CC6) {
177 err = tegra_cpuidle_coupled_barrier(dev);
178 if (err)
179 return err;
180 }
181
182 local_fiq_disable();
183 tegra_pm_set_cpu_in_lp2();
184 cpu_pm_enter();
185
186 ct_cpuidle_enter();
187
188 switch (index) {
189 case TEGRA_C7:
190 err = tegra_cpuidle_c7_enter();
191 break;
192
193 case TEGRA_CC6:
194 err = tegra_cpuidle_cc6_enter(cpu);
195 break;
196
197 default:
198 err = -EINVAL;
199 break;
200 }
201
202 ct_cpuidle_exit();
203
204 cpu_pm_exit();
205 tegra_pm_clear_cpu_in_lp2();
206 local_fiq_enable();
207
208 return err ?: index;
209 }
210
tegra_cpuidle_adjust_state_index(int index,unsigned int cpu)211 static int tegra_cpuidle_adjust_state_index(int index, unsigned int cpu)
212 {
213 /*
214 * On Tegra30 CPU0 can't be power-gated separately from secondary
215 * cores because it gates the whole CPU cluster.
216 */
217 if (cpu > 0 || index != TEGRA_C7 || tegra_get_chip_id() != TEGRA30)
218 return index;
219
220 /* put CPU0 into C1 if C7 is requested and secondaries are online */
221 if (!IS_ENABLED(CONFIG_PM_SLEEP) || num_online_cpus() > 1)
222 index = TEGRA_C1;
223 else
224 index = TEGRA_CC6;
225
226 return index;
227 }
228
tegra_cpuidle_enter(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)229 static __cpuidle int tegra_cpuidle_enter(struct cpuidle_device *dev,
230 struct cpuidle_driver *drv,
231 int index)
232 {
233 bool do_rcu = drv->states[index].flags & CPUIDLE_FLAG_RCU_IDLE;
234 unsigned int cpu = cpu_logical_map(dev->cpu);
235 int ret;
236
237 index = tegra_cpuidle_adjust_state_index(index, cpu);
238 if (dev->states_usage[index].disable)
239 return -1;
240
241 if (index == TEGRA_C1) {
242 if (do_rcu)
243 ct_cpuidle_enter();
244 ret = arm_cpuidle_simple_enter(dev, drv, index);
245 if (do_rcu)
246 ct_cpuidle_exit();
247 } else
248 ret = tegra_cpuidle_state_enter(dev, index, cpu);
249
250 if (ret < 0) {
251 if (ret != -EINTR || index != TEGRA_CC6)
252 pr_err_once("failed to enter state %d err: %d\n",
253 index, ret);
254 index = -1;
255 } else {
256 index = ret;
257 }
258
259 return index;
260 }
261
tegra114_enter_s2idle(struct cpuidle_device * dev,struct cpuidle_driver * drv,int index)262 static int tegra114_enter_s2idle(struct cpuidle_device *dev,
263 struct cpuidle_driver *drv,
264 int index)
265 {
266 tegra_cpuidle_enter(dev, drv, index);
267
268 return 0;
269 }
270
271 /*
272 * The previous versions of Tegra CPUIDLE driver used a different "legacy"
273 * terminology for naming of the idling states, while this driver uses the
274 * new terminology.
275 *
276 * Mapping of the old terms into the new ones:
277 *
278 * Old | New
279 * ---------
280 * LP3 | C1 (CPU core clock gating)
281 * LP2 | C7 (CPU core power gating)
282 * LP2 | CC6 (CPU cluster power gating)
283 *
284 * Note that that the older CPUIDLE driver versions didn't explicitly
285 * differentiate the LP2 states because these states either used the same
286 * code path or because CC6 wasn't supported.
287 */
288 static struct cpuidle_driver tegra_idle_driver = {
289 .name = "tegra_idle",
290 .states = {
291 [TEGRA_C1] = ARM_CPUIDLE_WFI_STATE_PWR(600),
292 [TEGRA_C7] = {
293 .enter = tegra_cpuidle_enter,
294 .exit_latency = 2000,
295 .target_residency = 2200,
296 .power_usage = 100,
297 .flags = CPUIDLE_FLAG_TIMER_STOP |
298 CPUIDLE_FLAG_RCU_IDLE,
299 .name = "C7",
300 .desc = "CPU core powered off",
301 },
302 [TEGRA_CC6] = {
303 .enter = tegra_cpuidle_enter,
304 .exit_latency = 5000,
305 .target_residency = 10000,
306 .power_usage = 0,
307 .flags = CPUIDLE_FLAG_TIMER_STOP |
308 CPUIDLE_FLAG_RCU_IDLE |
309 CPUIDLE_FLAG_COUPLED,
310 .name = "CC6",
311 .desc = "CPU cluster powered off",
312 },
313 },
314 .state_count = TEGRA_STATE_COUNT,
315 .safe_state_index = TEGRA_C1,
316 };
317
tegra_cpuidle_disable_state(enum tegra_state state)318 static inline void tegra_cpuidle_disable_state(enum tegra_state state)
319 {
320 cpuidle_driver_state_disabled(&tegra_idle_driver, state, true);
321 }
322
323 /*
324 * Tegra20 HW appears to have a bug such that PCIe device interrupts, whether
325 * they are legacy IRQs or MSI, are lost when CC6 is enabled. To work around
326 * this, simply disable CC6 if the PCI driver and DT node are both enabled.
327 */
tegra_cpuidle_pcie_irqs_in_use(void)328 void tegra_cpuidle_pcie_irqs_in_use(void)
329 {
330 struct cpuidle_state *state_cc6 = &tegra_idle_driver.states[TEGRA_CC6];
331
332 if ((state_cc6->flags & CPUIDLE_FLAG_UNUSABLE) ||
333 tegra_get_chip_id() != TEGRA20)
334 return;
335
336 pr_info("disabling CC6 state, since PCIe IRQs are in use\n");
337 tegra_cpuidle_disable_state(TEGRA_CC6);
338 }
339
tegra_cpuidle_setup_tegra114_c7_state(void)340 static void tegra_cpuidle_setup_tegra114_c7_state(void)
341 {
342 struct cpuidle_state *s = &tegra_idle_driver.states[TEGRA_C7];
343
344 s->enter_s2idle = tegra114_enter_s2idle;
345 s->target_residency = 1000;
346 s->exit_latency = 500;
347 }
348
tegra_cpuidle_probe(struct platform_device * pdev)349 static int tegra_cpuidle_probe(struct platform_device *pdev)
350 {
351 if (tegra_pmc_get_suspend_mode() == TEGRA_SUSPEND_NOT_READY)
352 return -EPROBE_DEFER;
353
354 /* LP2 could be disabled in device-tree */
355 if (tegra_pmc_get_suspend_mode() < TEGRA_SUSPEND_LP2)
356 tegra_cpuidle_disable_state(TEGRA_CC6);
357
358 /*
359 * Required suspend-resume functionality, which is provided by the
360 * Tegra-arch core and PMC driver, is unavailable if PM-sleep option
361 * is disabled.
362 */
363 if (!IS_ENABLED(CONFIG_PM_SLEEP)) {
364 tegra_cpuidle_disable_state(TEGRA_C7);
365 tegra_cpuidle_disable_state(TEGRA_CC6);
366 }
367
368 /*
369 * Generic WFI state (also known as C1 or LP3) and the coupled CPU
370 * cluster power-off (CC6 or LP2) states are common for all Tegra SoCs.
371 */
372 switch (tegra_get_chip_id()) {
373 case TEGRA20:
374 /* Tegra20 isn't capable to power-off individual CPU cores */
375 tegra_cpuidle_disable_state(TEGRA_C7);
376 break;
377
378 case TEGRA30:
379 break;
380
381 case TEGRA114:
382 case TEGRA124:
383 tegra_cpuidle_setup_tegra114_c7_state();
384
385 /* coupled CC6 (LP2) state isn't implemented yet */
386 tegra_cpuidle_disable_state(TEGRA_CC6);
387 break;
388
389 default:
390 return -EINVAL;
391 }
392
393 return cpuidle_register(&tegra_idle_driver, cpu_possible_mask);
394 }
395
396 static struct platform_driver tegra_cpuidle_driver = {
397 .probe = tegra_cpuidle_probe,
398 .driver = {
399 .name = "tegra-cpuidle",
400 },
401 };
402 builtin_platform_driver(tegra_cpuidle_driver);
403