xref: /openbmc/linux/arch/arm/common/mcpm_entry.c (revision 151f4e2b)
1 /*
2  * arch/arm/common/mcpm_entry.c -- entry point for multi-cluster PM
3  *
4  * Created by:  Nicolas Pitre, March 2012
5  * Copyright:   (C) 2012-2013  Linaro Limited
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License version 2 as
9  * published by the Free Software Foundation.
10  */
11 
12 #include <linux/export.h>
13 #include <linux/kernel.h>
14 #include <linux/init.h>
15 #include <linux/irqflags.h>
16 #include <linux/cpu_pm.h>
17 
18 #include <asm/mcpm.h>
19 #include <asm/cacheflush.h>
20 #include <asm/idmap.h>
21 #include <asm/cputype.h>
22 #include <asm/suspend.h>
23 
24 /*
25  * The public API for this code is documented in arch/arm/include/asm/mcpm.h.
26  * For a comprehensive description of the main algorithm used here, please
27  * see Documentation/arm/cluster-pm-race-avoidance.txt.
28  */
29 
30 struct sync_struct mcpm_sync;
31 
32 /*
33  * __mcpm_cpu_going_down: Indicates that the cpu is being torn down.
34  *    This must be called at the point of committing to teardown of a CPU.
35  *    The CPU cache (SCTRL.C bit) is expected to still be active.
36  */
37 static void __mcpm_cpu_going_down(unsigned int cpu, unsigned int cluster)
38 {
39 	mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_GOING_DOWN;
40 	sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
41 }
42 
43 /*
44  * __mcpm_cpu_down: Indicates that cpu teardown is complete and that the
45  *    cluster can be torn down without disrupting this CPU.
46  *    To avoid deadlocks, this must be called before a CPU is powered down.
47  *    The CPU cache (SCTRL.C bit) is expected to be off.
48  *    However L2 cache might or might not be active.
49  */
50 static void __mcpm_cpu_down(unsigned int cpu, unsigned int cluster)
51 {
52 	dmb();
53 	mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_DOWN;
54 	sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
55 	sev();
56 }
57 
58 /*
59  * __mcpm_outbound_leave_critical: Leave the cluster teardown critical section.
60  * @state: the final state of the cluster:
61  *     CLUSTER_UP: no destructive teardown was done and the cluster has been
62  *         restored to the previous state (CPU cache still active); or
63  *     CLUSTER_DOWN: the cluster has been torn-down, ready for power-off
64  *         (CPU cache disabled, L2 cache either enabled or disabled).
65  */
66 static void __mcpm_outbound_leave_critical(unsigned int cluster, int state)
67 {
68 	dmb();
69 	mcpm_sync.clusters[cluster].cluster = state;
70 	sync_cache_w(&mcpm_sync.clusters[cluster].cluster);
71 	sev();
72 }
73 
74 /*
75  * __mcpm_outbound_enter_critical: Enter the cluster teardown critical section.
76  * This function should be called by the last man, after local CPU teardown
77  * is complete.  CPU cache expected to be active.
78  *
79  * Returns:
80  *     false: the critical section was not entered because an inbound CPU was
81  *         observed, or the cluster is already being set up;
82  *     true: the critical section was entered: it is now safe to tear down the
83  *         cluster.
84  */
85 static bool __mcpm_outbound_enter_critical(unsigned int cpu, unsigned int cluster)
86 {
87 	unsigned int i;
88 	struct mcpm_sync_struct *c = &mcpm_sync.clusters[cluster];
89 
90 	/* Warn inbound CPUs that the cluster is being torn down: */
91 	c->cluster = CLUSTER_GOING_DOWN;
92 	sync_cache_w(&c->cluster);
93 
94 	/* Back out if the inbound cluster is already in the critical region: */
95 	sync_cache_r(&c->inbound);
96 	if (c->inbound == INBOUND_COMING_UP)
97 		goto abort;
98 
99 	/*
100 	 * Wait for all CPUs to get out of the GOING_DOWN state, so that local
101 	 * teardown is complete on each CPU before tearing down the cluster.
102 	 *
103 	 * If any CPU has been woken up again from the DOWN state, then we
104 	 * shouldn't be taking the cluster down at all: abort in that case.
105 	 */
106 	sync_cache_r(&c->cpus);
107 	for (i = 0; i < MAX_CPUS_PER_CLUSTER; i++) {
108 		int cpustate;
109 
110 		if (i == cpu)
111 			continue;
112 
113 		while (1) {
114 			cpustate = c->cpus[i].cpu;
115 			if (cpustate != CPU_GOING_DOWN)
116 				break;
117 
118 			wfe();
119 			sync_cache_r(&c->cpus[i].cpu);
120 		}
121 
122 		switch (cpustate) {
123 		case CPU_DOWN:
124 			continue;
125 
126 		default:
127 			goto abort;
128 		}
129 	}
130 
131 	return true;
132 
133 abort:
134 	__mcpm_outbound_leave_critical(cluster, CLUSTER_UP);
135 	return false;
136 }
137 
138 static int __mcpm_cluster_state(unsigned int cluster)
139 {
140 	sync_cache_r(&mcpm_sync.clusters[cluster].cluster);
141 	return mcpm_sync.clusters[cluster].cluster;
142 }
143 
144 extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
145 
146 void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr)
147 {
148 	unsigned long val = ptr ? __pa_symbol(ptr) : 0;
149 	mcpm_entry_vectors[cluster][cpu] = val;
150 	sync_cache_w(&mcpm_entry_vectors[cluster][cpu]);
151 }
152 
153 extern unsigned long mcpm_entry_early_pokes[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER][2];
154 
155 void mcpm_set_early_poke(unsigned cpu, unsigned cluster,
156 			 unsigned long poke_phys_addr, unsigned long poke_val)
157 {
158 	unsigned long *poke = &mcpm_entry_early_pokes[cluster][cpu][0];
159 	poke[0] = poke_phys_addr;
160 	poke[1] = poke_val;
161 	__sync_cache_range_w(poke, 2 * sizeof(*poke));
162 }
163 
164 static const struct mcpm_platform_ops *platform_ops;
165 
166 int __init mcpm_platform_register(const struct mcpm_platform_ops *ops)
167 {
168 	if (platform_ops)
169 		return -EBUSY;
170 	platform_ops = ops;
171 	return 0;
172 }
173 
174 bool mcpm_is_available(void)
175 {
176 	return (platform_ops) ? true : false;
177 }
178 EXPORT_SYMBOL_GPL(mcpm_is_available);
179 
180 /*
181  * We can't use regular spinlocks. In the switcher case, it is possible
182  * for an outbound CPU to call power_down() after its inbound counterpart
183  * is already live using the same logical CPU number which trips lockdep
184  * debugging.
185  */
186 static arch_spinlock_t mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED;
187 
188 static int mcpm_cpu_use_count[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
189 
190 static inline bool mcpm_cluster_unused(unsigned int cluster)
191 {
192 	int i, cnt;
193 	for (i = 0, cnt = 0; i < MAX_CPUS_PER_CLUSTER; i++)
194 		cnt |= mcpm_cpu_use_count[cluster][i];
195 	return !cnt;
196 }
197 
198 int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster)
199 {
200 	bool cpu_is_down, cluster_is_down;
201 	int ret = 0;
202 
203 	pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
204 	if (!platform_ops)
205 		return -EUNATCH; /* try not to shadow power_up errors */
206 	might_sleep();
207 
208 	/*
209 	 * Since this is called with IRQs enabled, and no arch_spin_lock_irq
210 	 * variant exists, we need to disable IRQs manually here.
211 	 */
212 	local_irq_disable();
213 	arch_spin_lock(&mcpm_lock);
214 
215 	cpu_is_down = !mcpm_cpu_use_count[cluster][cpu];
216 	cluster_is_down = mcpm_cluster_unused(cluster);
217 
218 	mcpm_cpu_use_count[cluster][cpu]++;
219 	/*
220 	 * The only possible values are:
221 	 * 0 = CPU down
222 	 * 1 = CPU (still) up
223 	 * 2 = CPU requested to be up before it had a chance
224 	 *     to actually make itself down.
225 	 * Any other value is a bug.
226 	 */
227 	BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 1 &&
228 	       mcpm_cpu_use_count[cluster][cpu] != 2);
229 
230 	if (cluster_is_down)
231 		ret = platform_ops->cluster_powerup(cluster);
232 	if (cpu_is_down && !ret)
233 		ret = platform_ops->cpu_powerup(cpu, cluster);
234 
235 	arch_spin_unlock(&mcpm_lock);
236 	local_irq_enable();
237 	return ret;
238 }
239 
240 typedef typeof(cpu_reset) phys_reset_t;
241 
242 void mcpm_cpu_power_down(void)
243 {
244 	unsigned int mpidr, cpu, cluster;
245 	bool cpu_going_down, last_man;
246 	phys_reset_t phys_reset;
247 
248 	mpidr = read_cpuid_mpidr();
249 	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
250 	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
251 	pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
252 	if (WARN_ON_ONCE(!platform_ops))
253 	       return;
254 	BUG_ON(!irqs_disabled());
255 
256 	setup_mm_for_reboot();
257 
258 	__mcpm_cpu_going_down(cpu, cluster);
259 	arch_spin_lock(&mcpm_lock);
260 	BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);
261 
262 	mcpm_cpu_use_count[cluster][cpu]--;
263 	BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 0 &&
264 	       mcpm_cpu_use_count[cluster][cpu] != 1);
265 	cpu_going_down = !mcpm_cpu_use_count[cluster][cpu];
266 	last_man = mcpm_cluster_unused(cluster);
267 
268 	if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
269 		platform_ops->cpu_powerdown_prepare(cpu, cluster);
270 		platform_ops->cluster_powerdown_prepare(cluster);
271 		arch_spin_unlock(&mcpm_lock);
272 		platform_ops->cluster_cache_disable();
273 		__mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
274 	} else {
275 		if (cpu_going_down)
276 			platform_ops->cpu_powerdown_prepare(cpu, cluster);
277 		arch_spin_unlock(&mcpm_lock);
278 		/*
279 		 * If cpu_going_down is false here, that means a power_up
280 		 * request raced ahead of us.  Even if we do not want to
281 		 * shut this CPU down, the caller still expects execution
282 		 * to return through the system resume entry path, like
283 		 * when the WFI is aborted due to a new IRQ or the like..
284 		 * So let's continue with cache cleaning in all cases.
285 		 */
286 		platform_ops->cpu_cache_disable();
287 	}
288 
289 	__mcpm_cpu_down(cpu, cluster);
290 
291 	/* Now we are prepared for power-down, do it: */
292 	if (cpu_going_down)
293 		wfi();
294 
295 	/*
296 	 * It is possible for a power_up request to happen concurrently
297 	 * with a power_down request for the same CPU. In this case the
298 	 * CPU might not be able to actually enter a powered down state
299 	 * with the WFI instruction if the power_up request has removed
300 	 * the required reset condition.  We must perform a re-entry in
301 	 * the kernel as if the power_up method just had deasserted reset
302 	 * on the CPU.
303 	 */
304 	phys_reset = (phys_reset_t)(unsigned long)__pa_symbol(cpu_reset);
305 	phys_reset(__pa_symbol(mcpm_entry_point), false);
306 
307 	/* should never get here */
308 	BUG();
309 }
310 
311 int mcpm_wait_for_cpu_powerdown(unsigned int cpu, unsigned int cluster)
312 {
313 	int ret;
314 
315 	if (WARN_ON_ONCE(!platform_ops || !platform_ops->wait_for_powerdown))
316 		return -EUNATCH;
317 
318 	ret = platform_ops->wait_for_powerdown(cpu, cluster);
319 	if (ret)
320 		pr_warn("%s: cpu %u, cluster %u failed to power down (%d)\n",
321 			__func__, cpu, cluster, ret);
322 
323 	return ret;
324 }
325 
326 void mcpm_cpu_suspend(void)
327 {
328 	if (WARN_ON_ONCE(!platform_ops))
329 		return;
330 
331 	/* Some platforms might have to enable special resume modes, etc. */
332 	if (platform_ops->cpu_suspend_prepare) {
333 		unsigned int mpidr = read_cpuid_mpidr();
334 		unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
335 		unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
336 		arch_spin_lock(&mcpm_lock);
337 		platform_ops->cpu_suspend_prepare(cpu, cluster);
338 		arch_spin_unlock(&mcpm_lock);
339 	}
340 	mcpm_cpu_power_down();
341 }
342 
343 int mcpm_cpu_powered_up(void)
344 {
345 	unsigned int mpidr, cpu, cluster;
346 	bool cpu_was_down, first_man;
347 	unsigned long flags;
348 
349 	if (!platform_ops)
350 		return -EUNATCH;
351 
352 	mpidr = read_cpuid_mpidr();
353 	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
354 	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
355 	local_irq_save(flags);
356 	arch_spin_lock(&mcpm_lock);
357 
358 	cpu_was_down = !mcpm_cpu_use_count[cluster][cpu];
359 	first_man = mcpm_cluster_unused(cluster);
360 
361 	if (first_man && platform_ops->cluster_is_up)
362 		platform_ops->cluster_is_up(cluster);
363 	if (cpu_was_down)
364 		mcpm_cpu_use_count[cluster][cpu] = 1;
365 	if (platform_ops->cpu_is_up)
366 		platform_ops->cpu_is_up(cpu, cluster);
367 
368 	arch_spin_unlock(&mcpm_lock);
369 	local_irq_restore(flags);
370 
371 	return 0;
372 }
373 
374 #ifdef CONFIG_ARM_CPU_SUSPEND
375 
376 static int __init nocache_trampoline(unsigned long _arg)
377 {
378 	void (*cache_disable)(void) = (void *)_arg;
379 	unsigned int mpidr = read_cpuid_mpidr();
380 	unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
381 	unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
382 	phys_reset_t phys_reset;
383 
384 	mcpm_set_entry_vector(cpu, cluster, cpu_resume_no_hyp);
385 	setup_mm_for_reboot();
386 
387 	__mcpm_cpu_going_down(cpu, cluster);
388 	BUG_ON(!__mcpm_outbound_enter_critical(cpu, cluster));
389 	cache_disable();
390 	__mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
391 	__mcpm_cpu_down(cpu, cluster);
392 
393 	phys_reset = (phys_reset_t)(unsigned long)__pa_symbol(cpu_reset);
394 	phys_reset(__pa_symbol(mcpm_entry_point), false);
395 	BUG();
396 }
397 
398 int __init mcpm_loopback(void (*cache_disable)(void))
399 {
400 	int ret;
401 
402 	/*
403 	 * We're going to soft-restart the current CPU through the
404 	 * low-level MCPM code by leveraging the suspend/resume
405 	 * infrastructure. Let's play it safe by using cpu_pm_enter()
406 	 * in case the CPU init code path resets the VFP or similar.
407 	 */
408 	local_irq_disable();
409 	local_fiq_disable();
410 	ret = cpu_pm_enter();
411 	if (!ret) {
412 		ret = cpu_suspend((unsigned long)cache_disable, nocache_trampoline);
413 		cpu_pm_exit();
414 	}
415 	local_fiq_enable();
416 	local_irq_enable();
417 	if (ret)
418 		pr_err("%s returned %d\n", __func__, ret);
419 	return ret;
420 }
421 
422 #endif
423 
424 extern unsigned long mcpm_power_up_setup_phys;
425 
426 int __init mcpm_sync_init(
427 	void (*power_up_setup)(unsigned int affinity_level))
428 {
429 	unsigned int i, j, mpidr, this_cluster;
430 
431 	BUILD_BUG_ON(MCPM_SYNC_CLUSTER_SIZE * MAX_NR_CLUSTERS != sizeof mcpm_sync);
432 	BUG_ON((unsigned long)&mcpm_sync & (__CACHE_WRITEBACK_GRANULE - 1));
433 
434 	/*
435 	 * Set initial CPU and cluster states.
436 	 * Only one cluster is assumed to be active at this point.
437 	 */
438 	for (i = 0; i < MAX_NR_CLUSTERS; i++) {
439 		mcpm_sync.clusters[i].cluster = CLUSTER_DOWN;
440 		mcpm_sync.clusters[i].inbound = INBOUND_NOT_COMING_UP;
441 		for (j = 0; j < MAX_CPUS_PER_CLUSTER; j++)
442 			mcpm_sync.clusters[i].cpus[j].cpu = CPU_DOWN;
443 	}
444 	mpidr = read_cpuid_mpidr();
445 	this_cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
446 	for_each_online_cpu(i) {
447 		mcpm_cpu_use_count[this_cluster][i] = 1;
448 		mcpm_sync.clusters[this_cluster].cpus[i].cpu = CPU_UP;
449 	}
450 	mcpm_sync.clusters[this_cluster].cluster = CLUSTER_UP;
451 	sync_cache_w(&mcpm_sync);
452 
453 	if (power_up_setup) {
454 		mcpm_power_up_setup_phys = __pa_symbol(power_up_setup);
455 		sync_cache_w(&mcpm_power_up_setup_phys);
456 	}
457 
458 	return 0;
459 }
460