1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * CPU idle driver for Tegra CPUs
4  *
5  * Copyright (c) 2010-2013, NVIDIA Corporation.
6  * Copyright (c) 2011 Google, Inc.
7  * Author: Colin Cross <ccross@android.com>
8  *         Gary King <gking@nvidia.com>
9  *
10  * Rework for 3.3 by Peter De Schrijver <pdeschrijver@nvidia.com>
11  *
12  * Tegra20/124 driver unification by Dmitry Osipenko <digetx@gmail.com>
13  */
14 
15 #define pr_fmt(fmt)	"tegra-cpuidle: " fmt
16 
17 #include <linux/atomic.h>
18 #include <linux/cpuidle.h>
19 #include <linux/cpumask.h>
20 #include <linux/cpu_pm.h>
21 #include <linux/delay.h>
22 #include <linux/errno.h>
23 #include <linux/platform_device.h>
24 #include <linux/types.h>
25 
26 #include <linux/clk/tegra.h>
27 #include <linux/firmware/trusted_foundations.h>
28 
29 #include <soc/tegra/cpuidle.h>
30 #include <soc/tegra/flowctrl.h>
31 #include <soc/tegra/fuse.h>
32 #include <soc/tegra/irq.h>
33 #include <soc/tegra/pm.h>
34 #include <soc/tegra/pmc.h>
35 
36 #include <asm/cpuidle.h>
37 #include <asm/firmware.h>
38 #include <asm/smp_plat.h>
39 #include <asm/suspend.h>
40 
41 enum tegra_state {
42 	TEGRA_C1,
43 	TEGRA_C7,
44 	TEGRA_CC6,
45 	TEGRA_STATE_COUNT,
46 };
47 
48 static atomic_t tegra_idle_barrier;
49 static atomic_t tegra_abort_flag;
50 
51 static inline bool tegra_cpuidle_using_firmware(void)
52 {
53 	return firmware_ops->prepare_idle && firmware_ops->do_idle;
54 }
55 
56 static void tegra_cpuidle_report_cpus_state(void)
57 {
58 	unsigned long cpu, lcpu, csr;
59 
60 	for_each_cpu(lcpu, cpu_possible_mask) {
61 		cpu = cpu_logical_map(lcpu);
62 		csr = flowctrl_read_cpu_csr(cpu);
63 
64 		pr_err("cpu%lu: online=%d flowctrl_csr=0x%08lx\n",
65 		       cpu, cpu_online(lcpu), csr);
66 	}
67 }
68 
69 static int tegra_cpuidle_wait_for_secondary_cpus_parking(void)
70 {
71 	unsigned int retries = 3;
72 
73 	while (retries--) {
74 		unsigned int delay_us = 10;
75 		unsigned int timeout_us = 500 * 1000 / delay_us;
76 
77 		/*
78 		 * The primary CPU0 core shall wait for the secondaries
79 		 * shutdown in order to power-off CPU's cluster safely.
80 		 * The timeout value depends on the current CPU frequency,
81 		 * it takes about 40-150us in average and over 1000us in
82 		 * a worst case scenario.
83 		 */
84 		do {
85 			if (tegra_cpu_rail_off_ready())
86 				return 0;
87 
88 			udelay(delay_us);
89 
90 		} while (timeout_us--);
91 
92 		pr_err("secondary CPU taking too long to park\n");
93 
94 		tegra_cpuidle_report_cpus_state();
95 	}
96 
97 	pr_err("timed out waiting secondaries to park\n");
98 
99 	return -ETIMEDOUT;
100 }
101 
102 static void tegra_cpuidle_unpark_secondary_cpus(void)
103 {
104 	unsigned int cpu, lcpu;
105 
106 	for_each_cpu(lcpu, cpu_online_mask) {
107 		cpu = cpu_logical_map(lcpu);
108 
109 		if (cpu > 0) {
110 			tegra_enable_cpu_clock(cpu);
111 			tegra_cpu_out_of_reset(cpu);
112 			flowctrl_write_cpu_halt(cpu, 0);
113 		}
114 	}
115 }
116 
117 static int tegra_cpuidle_cc6_enter(unsigned int cpu)
118 {
119 	int ret;
120 
121 	if (cpu > 0) {
122 		ret = cpu_suspend(cpu, tegra_pm_park_secondary_cpu);
123 	} else {
124 		ret = tegra_cpuidle_wait_for_secondary_cpus_parking();
125 		if (!ret)
126 			ret = tegra_pm_enter_lp2();
127 
128 		tegra_cpuidle_unpark_secondary_cpus();
129 	}
130 
131 	return ret;
132 }
133 
134 static int tegra_cpuidle_c7_enter(void)
135 {
136 	int err;
137 
138 	if (tegra_cpuidle_using_firmware()) {
139 		err = call_firmware_op(prepare_idle, TF_PM_MODE_LP2_NOFLUSH_L2);
140 		if (err)
141 			return err;
142 
143 		return call_firmware_op(do_idle, 0);
144 	}
145 
146 	return cpu_suspend(0, tegra30_pm_secondary_cpu_suspend);
147 }
148 
149 static int tegra_cpuidle_coupled_barrier(struct cpuidle_device *dev)
150 {
151 	if (tegra_pending_sgi()) {
152 		/*
153 		 * CPU got local interrupt that will be lost after GIC's
154 		 * shutdown because GIC driver doesn't save/restore the
155 		 * pending SGI state across CPU cluster PM.  Abort and retry
156 		 * next time.
157 		 */
158 		atomic_set(&tegra_abort_flag, 1);
159 	}
160 
161 	cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier);
162 
163 	if (atomic_read(&tegra_abort_flag)) {
164 		cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier);
165 		atomic_set(&tegra_abort_flag, 0);
166 		return -EINTR;
167 	}
168 
169 	return 0;
170 }
171 
172 static int tegra_cpuidle_state_enter(struct cpuidle_device *dev,
173 				     int index, unsigned int cpu)
174 {
175 	int ret;
176 
177 	/*
178 	 * CC6 state is the "CPU cluster power-off" state.  In order to
179 	 * enter this state, at first the secondary CPU cores need to be
180 	 * parked into offline mode, then the last CPU should clean out
181 	 * remaining dirty cache lines into DRAM and trigger Flow Controller
182 	 * logic that turns off the cluster's power domain (which includes
183 	 * CPU cores, GIC and L2 cache).
184 	 */
185 	if (index == TEGRA_CC6) {
186 		ret = tegra_cpuidle_coupled_barrier(dev);
187 		if (ret)
188 			return ret;
189 	}
190 
191 	local_fiq_disable();
192 	tegra_pm_set_cpu_in_lp2();
193 	cpu_pm_enter();
194 
195 	switch (index) {
196 	case TEGRA_C7:
197 		ret = tegra_cpuidle_c7_enter();
198 		break;
199 
200 	case TEGRA_CC6:
201 		ret = tegra_cpuidle_cc6_enter(cpu);
202 		break;
203 
204 	default:
205 		ret = -EINVAL;
206 		break;
207 	}
208 
209 	cpu_pm_exit();
210 	tegra_pm_clear_cpu_in_lp2();
211 	local_fiq_enable();
212 
213 	return ret;
214 }
215 
216 static int tegra_cpuidle_adjust_state_index(int index, unsigned int cpu)
217 {
218 	/*
219 	 * On Tegra30 CPU0 can't be power-gated separately from secondary
220 	 * cores because it gates the whole CPU cluster.
221 	 */
222 	if (cpu > 0 || index != TEGRA_C7 || tegra_get_chip_id() != TEGRA30)
223 		return index;
224 
225 	/* put CPU0 into C1 if C7 is requested and secondaries are online */
226 	if (!IS_ENABLED(CONFIG_PM_SLEEP) || num_online_cpus() > 1)
227 		index = TEGRA_C1;
228 	else
229 		index = TEGRA_CC6;
230 
231 	return index;
232 }
233 
234 static int tegra_cpuidle_enter(struct cpuidle_device *dev,
235 			       struct cpuidle_driver *drv,
236 			       int index)
237 {
238 	unsigned int cpu = cpu_logical_map(dev->cpu);
239 	int err;
240 
241 	index = tegra_cpuidle_adjust_state_index(index, cpu);
242 	if (dev->states_usage[index].disable)
243 		return -1;
244 
245 	if (index == TEGRA_C1)
246 		err = arm_cpuidle_simple_enter(dev, drv, index);
247 	else
248 		err = tegra_cpuidle_state_enter(dev, index, cpu);
249 
250 	if (err && (err != -EINTR || index != TEGRA_CC6))
251 		pr_err_once("failed to enter state %d err: %d\n", index, err);
252 
253 	return err ? -1 : index;
254 }
255 
256 static void tegra114_enter_s2idle(struct cpuidle_device *dev,
257 				  struct cpuidle_driver *drv,
258 				  int index)
259 {
260 	tegra_cpuidle_enter(dev, drv, index);
261 }
262 
263 /*
264  * The previous versions of Tegra CPUIDLE driver used a different "legacy"
265  * terminology for naming of the idling states, while this driver uses the
266  * new terminology.
267  *
268  * Mapping of the old terms into the new ones:
269  *
270  * Old | New
271  * ---------
272  * LP3 | C1	(CPU core clock gating)
273  * LP2 | C7	(CPU core power gating)
274  * LP2 | CC6	(CPU cluster power gating)
275  *
276  * Note that that the older CPUIDLE driver versions didn't explicitly
277  * differentiate the LP2 states because these states either used the same
278  * code path or because CC6 wasn't supported.
279  */
280 static struct cpuidle_driver tegra_idle_driver = {
281 	.name = "tegra_idle",
282 	.states = {
283 		[TEGRA_C1] = ARM_CPUIDLE_WFI_STATE_PWR(600),
284 		[TEGRA_C7] = {
285 			.enter			= tegra_cpuidle_enter,
286 			.exit_latency		= 2000,
287 			.target_residency	= 2200,
288 			.power_usage		= 100,
289 			.flags			= CPUIDLE_FLAG_TIMER_STOP,
290 			.name			= "C7",
291 			.desc			= "CPU core powered off",
292 		},
293 		[TEGRA_CC6] = {
294 			.enter			= tegra_cpuidle_enter,
295 			.exit_latency		= 5000,
296 			.target_residency	= 10000,
297 			.power_usage		= 0,
298 			.flags			= CPUIDLE_FLAG_TIMER_STOP |
299 						  CPUIDLE_FLAG_COUPLED,
300 			.name			= "CC6",
301 			.desc			= "CPU cluster powered off",
302 		},
303 	},
304 	.state_count = TEGRA_STATE_COUNT,
305 	.safe_state_index = TEGRA_C1,
306 };
307 
308 static inline void tegra_cpuidle_disable_state(enum tegra_state state)
309 {
310 	cpuidle_driver_state_disabled(&tegra_idle_driver, state, true);
311 }
312 
313 /*
314  * Tegra20 HW appears to have a bug such that PCIe device interrupts, whether
315  * they are legacy IRQs or MSI, are lost when CC6 is enabled.  To work around
316  * this, simply disable CC6 if the PCI driver and DT node are both enabled.
317  */
318 void tegra_cpuidle_pcie_irqs_in_use(void)
319 {
320 	struct cpuidle_state *state_cc6 = &tegra_idle_driver.states[TEGRA_CC6];
321 
322 	if ((state_cc6->flags & CPUIDLE_FLAG_UNUSABLE) ||
323 	    tegra_get_chip_id() != TEGRA20)
324 		return;
325 
326 	pr_info("disabling CC6 state, since PCIe IRQs are in use\n");
327 	tegra_cpuidle_disable_state(TEGRA_CC6);
328 }
329 
330 static void tegra_cpuidle_setup_tegra114_c7_state(void)
331 {
332 	struct cpuidle_state *s = &tegra_idle_driver.states[TEGRA_C7];
333 
334 	s->enter_s2idle = tegra114_enter_s2idle;
335 	s->target_residency = 1000;
336 	s->exit_latency = 500;
337 }
338 
339 static int tegra_cpuidle_probe(struct platform_device *pdev)
340 {
341 	/* LP2 could be disabled in device-tree */
342 	if (tegra_pmc_get_suspend_mode() < TEGRA_SUSPEND_LP2)
343 		tegra_cpuidle_disable_state(TEGRA_CC6);
344 
345 	/*
346 	 * Required suspend-resume functionality, which is provided by the
347 	 * Tegra-arch core and PMC driver, is unavailable if PM-sleep option
348 	 * is disabled.
349 	 */
350 	if (!IS_ENABLED(CONFIG_PM_SLEEP)) {
351 		if (!tegra_cpuidle_using_firmware())
352 			tegra_cpuidle_disable_state(TEGRA_C7);
353 
354 		tegra_cpuidle_disable_state(TEGRA_CC6);
355 	}
356 
357 	/*
358 	 * Generic WFI state (also known as C1 or LP3) and the coupled CPU
359 	 * cluster power-off (CC6 or LP2) states are common for all Tegra SoCs.
360 	 */
361 	switch (tegra_get_chip_id()) {
362 	case TEGRA20:
363 		/* Tegra20 isn't capable to power-off individual CPU cores */
364 		tegra_cpuidle_disable_state(TEGRA_C7);
365 		break;
366 
367 	case TEGRA30:
368 		tegra_cpuidle_disable_state(TEGRA_CC6);
369 		break;
370 
371 	case TEGRA114:
372 	case TEGRA124:
373 		tegra_cpuidle_setup_tegra114_c7_state();
374 
375 		/* coupled CC6 (LP2) state isn't implemented yet */
376 		tegra_cpuidle_disable_state(TEGRA_CC6);
377 		break;
378 
379 	default:
380 		return -EINVAL;
381 	}
382 
383 	return cpuidle_register(&tegra_idle_driver, cpu_possible_mask);
384 }
385 
386 static struct platform_driver tegra_cpuidle_driver = {
387 	.probe = tegra_cpuidle_probe,
388 	.driver = {
389 		.name = "tegra-cpuidle",
390 	},
391 };
392 builtin_platform_driver(tegra_cpuidle_driver);
393