1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * drivers/base/power/domain_governor.c - Governors for device PM domains.
4  *
5  * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp.
6  */
7 #include <linux/kernel.h>
8 #include <linux/pm_domain.h>
9 #include <linux/pm_qos.h>
10 #include <linux/hrtimer.h>
11 #include <linux/cpuidle.h>
12 #include <linux/cpumask.h>
13 #include <linux/ktime.h>
14 
15 static int dev_update_qos_constraint(struct device *dev, void *data)
16 {
17 	s64 *constraint_ns_p = data;
18 	s64 constraint_ns;
19 
20 	if (dev->power.subsys_data && dev->power.subsys_data->domain_data) {
21 		struct gpd_timing_data *td = dev_gpd_data(dev)->td;
22 
23 		/*
24 		 * Only take suspend-time QoS constraints of devices into
25 		 * account, because constraints updated after the device has
26 		 * been suspended are not guaranteed to be taken into account
27 		 * anyway.  In order for them to take effect, the device has to
28 		 * be resumed and suspended again.
29 		 */
30 		constraint_ns = td ? td->effective_constraint_ns :
31 				PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS;
32 	} else {
33 		/*
34 		 * The child is not in a domain and there's no info on its
35 		 * suspend/resume latencies, so assume them to be negligible and
36 		 * take its current PM QoS constraint (that's the only thing
37 		 * known at this point anyway).
38 		 */
39 		constraint_ns = dev_pm_qos_read_value(dev, DEV_PM_QOS_RESUME_LATENCY);
40 		constraint_ns *= NSEC_PER_USEC;
41 	}
42 
43 	if (constraint_ns < *constraint_ns_p)
44 		*constraint_ns_p = constraint_ns;
45 
46 	return 0;
47 }
48 
49 /**
50  * default_suspend_ok - Default PM domain governor routine to suspend devices.
51  * @dev: Device to check.
52  */
53 static bool default_suspend_ok(struct device *dev)
54 {
55 	struct gpd_timing_data *td = dev_gpd_data(dev)->td;
56 	unsigned long flags;
57 	s64 constraint_ns;
58 
59 	dev_dbg(dev, "%s()\n", __func__);
60 
61 	spin_lock_irqsave(&dev->power.lock, flags);
62 
63 	if (!td->constraint_changed) {
64 		bool ret = td->cached_suspend_ok;
65 
66 		spin_unlock_irqrestore(&dev->power.lock, flags);
67 		return ret;
68 	}
69 	td->constraint_changed = false;
70 	td->cached_suspend_ok = false;
71 	td->effective_constraint_ns = 0;
72 	constraint_ns = __dev_pm_qos_resume_latency(dev);
73 
74 	spin_unlock_irqrestore(&dev->power.lock, flags);
75 
76 	if (constraint_ns == 0)
77 		return false;
78 
79 	constraint_ns *= NSEC_PER_USEC;
80 	/*
81 	 * We can walk the children without any additional locking, because
82 	 * they all have been suspended at this point and their
83 	 * effective_constraint_ns fields won't be modified in parallel with us.
84 	 */
85 	if (!dev->power.ignore_children)
86 		device_for_each_child(dev, &constraint_ns,
87 				      dev_update_qos_constraint);
88 
89 	if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS) {
90 		/* "No restriction", so the device is allowed to suspend. */
91 		td->effective_constraint_ns = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS;
92 		td->cached_suspend_ok = true;
93 	} else if (constraint_ns == 0) {
94 		/*
95 		 * This triggers if one of the children that don't belong to a
96 		 * domain has a zero PM QoS constraint and it's better not to
97 		 * suspend then.  effective_constraint_ns is zero already and
98 		 * cached_suspend_ok is false, so bail out.
99 		 */
100 		return false;
101 	} else {
102 		constraint_ns -= td->suspend_latency_ns +
103 				td->resume_latency_ns;
104 		/*
105 		 * effective_constraint_ns is zero already and cached_suspend_ok
106 		 * is false, so if the computed value is not positive, return
107 		 * right away.
108 		 */
109 		if (constraint_ns <= 0)
110 			return false;
111 
112 		td->effective_constraint_ns = constraint_ns;
113 		td->cached_suspend_ok = true;
114 	}
115 
116 	/*
117 	 * The children have been suspended already, so we don't need to take
118 	 * their suspend latencies into account here.
119 	 */
120 	return td->cached_suspend_ok;
121 }
122 
123 static void update_domain_next_wakeup(struct generic_pm_domain *genpd, ktime_t now)
124 {
125 	ktime_t domain_wakeup = KTIME_MAX;
126 	ktime_t next_wakeup;
127 	struct pm_domain_data *pdd;
128 	struct gpd_link *link;
129 
130 	if (!(genpd->flags & GENPD_FLAG_MIN_RESIDENCY))
131 		return;
132 
133 	/*
134 	 * Devices that have a predictable wakeup pattern, may specify
135 	 * their next wakeup. Let's find the next wakeup from all the
136 	 * devices attached to this domain and from all the sub-domains.
137 	 * It is possible that component's a next wakeup may have become
138 	 * stale when we read that here. We will ignore to ensure the domain
139 	 * is able to enter its optimal idle state.
140 	 */
141 	list_for_each_entry(pdd, &genpd->dev_list, list_node) {
142 		next_wakeup = to_gpd_data(pdd)->td->next_wakeup;
143 		if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now))
144 			if (ktime_before(next_wakeup, domain_wakeup))
145 				domain_wakeup = next_wakeup;
146 	}
147 
148 	list_for_each_entry(link, &genpd->parent_links, parent_node) {
149 		struct genpd_governor_data *cgd = link->child->gd;
150 
151 		next_wakeup = cgd ? cgd->next_wakeup : KTIME_MAX;
152 		if (next_wakeup != KTIME_MAX && !ktime_before(next_wakeup, now))
153 			if (ktime_before(next_wakeup, domain_wakeup))
154 				domain_wakeup = next_wakeup;
155 	}
156 
157 	genpd->gd->next_wakeup = domain_wakeup;
158 }
159 
160 static bool next_wakeup_allows_state(struct generic_pm_domain *genpd,
161 				     unsigned int state, ktime_t now)
162 {
163 	ktime_t domain_wakeup = genpd->gd->next_wakeup;
164 	s64 idle_time_ns, min_sleep_ns;
165 
166 	min_sleep_ns = genpd->states[state].power_off_latency_ns +
167 		       genpd->states[state].residency_ns;
168 
169 	idle_time_ns = ktime_to_ns(ktime_sub(domain_wakeup, now));
170 
171 	return idle_time_ns >= min_sleep_ns;
172 }
173 
174 static bool __default_power_down_ok(struct dev_pm_domain *pd,
175 				     unsigned int state)
176 {
177 	struct generic_pm_domain *genpd = pd_to_genpd(pd);
178 	struct gpd_link *link;
179 	struct pm_domain_data *pdd;
180 	s64 min_off_time_ns;
181 	s64 off_on_time_ns;
182 
183 	off_on_time_ns = genpd->states[state].power_off_latency_ns +
184 		genpd->states[state].power_on_latency_ns;
185 
186 	min_off_time_ns = -1;
187 	/*
188 	 * Check if subdomains can be off for enough time.
189 	 *
190 	 * All subdomains have been powered off already at this point.
191 	 */
192 	list_for_each_entry(link, &genpd->parent_links, parent_node) {
193 		struct genpd_governor_data *cgd = link->child->gd;
194 
195 		s64 sd_max_off_ns = cgd ? cgd->max_off_time_ns : -1;
196 
197 		if (sd_max_off_ns < 0)
198 			continue;
199 
200 		/*
201 		 * Check if the subdomain is allowed to be off long enough for
202 		 * the current domain to turn off and on (that's how much time
203 		 * it will have to wait worst case).
204 		 */
205 		if (sd_max_off_ns <= off_on_time_ns)
206 			return false;
207 
208 		if (min_off_time_ns > sd_max_off_ns || min_off_time_ns < 0)
209 			min_off_time_ns = sd_max_off_ns;
210 	}
211 
212 	/*
213 	 * Check if the devices in the domain can be off enough time.
214 	 */
215 	list_for_each_entry(pdd, &genpd->dev_list, list_node) {
216 		struct gpd_timing_data *td;
217 		s64 constraint_ns;
218 
219 		/*
220 		 * Check if the device is allowed to be off long enough for the
221 		 * domain to turn off and on (that's how much time it will
222 		 * have to wait worst case).
223 		 */
224 		td = to_gpd_data(pdd)->td;
225 		constraint_ns = td->effective_constraint_ns;
226 		/*
227 		 * Zero means "no suspend at all" and this runs only when all
228 		 * devices in the domain are suspended, so it must be positive.
229 		 */
230 		if (constraint_ns == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT_NS)
231 			continue;
232 
233 		if (constraint_ns <= off_on_time_ns)
234 			return false;
235 
236 		if (min_off_time_ns > constraint_ns || min_off_time_ns < 0)
237 			min_off_time_ns = constraint_ns;
238 	}
239 
240 	/*
241 	 * If the computed minimum device off time is negative, there are no
242 	 * latency constraints, so the domain can spend arbitrary time in the
243 	 * "off" state.
244 	 */
245 	if (min_off_time_ns < 0)
246 		return true;
247 
248 	/*
249 	 * The difference between the computed minimum subdomain or device off
250 	 * time and the time needed to turn the domain on is the maximum
251 	 * theoretical time this domain can spend in the "off" state.
252 	 */
253 	genpd->gd->max_off_time_ns = min_off_time_ns -
254 		genpd->states[state].power_on_latency_ns;
255 	return true;
256 }
257 
258 /**
259  * _default_power_down_ok - Default generic PM domain power off governor routine.
260  * @pd: PM domain to check.
261  * @now: current ktime.
262  *
263  * This routine must be executed under the PM domain's lock.
264  */
265 static bool _default_power_down_ok(struct dev_pm_domain *pd, ktime_t now)
266 {
267 	struct generic_pm_domain *genpd = pd_to_genpd(pd);
268 	struct genpd_governor_data *gd = genpd->gd;
269 	int state_idx = genpd->state_count - 1;
270 	struct gpd_link *link;
271 
272 	/*
273 	 * Find the next wakeup from devices that can determine their own wakeup
274 	 * to find when the domain would wakeup and do it for every device down
275 	 * the hierarchy. It is not worth while to sleep if the state's residency
276 	 * cannot be met.
277 	 */
278 	update_domain_next_wakeup(genpd, now);
279 	if ((genpd->flags & GENPD_FLAG_MIN_RESIDENCY) && (gd->next_wakeup != KTIME_MAX)) {
280 		/* Let's find out the deepest domain idle state, the devices prefer */
281 		while (state_idx >= 0) {
282 			if (next_wakeup_allows_state(genpd, state_idx, now)) {
283 				gd->max_off_time_changed = true;
284 				break;
285 			}
286 			state_idx--;
287 		}
288 
289 		if (state_idx < 0) {
290 			state_idx = 0;
291 			gd->cached_power_down_ok = false;
292 			goto done;
293 		}
294 	}
295 
296 	if (!gd->max_off_time_changed) {
297 		genpd->state_idx = gd->cached_power_down_state_idx;
298 		return gd->cached_power_down_ok;
299 	}
300 
301 	/*
302 	 * We have to invalidate the cached results for the parents, so
303 	 * use the observation that default_power_down_ok() is not
304 	 * going to be called for any parent until this instance
305 	 * returns.
306 	 */
307 	list_for_each_entry(link, &genpd->child_links, child_node) {
308 		struct genpd_governor_data *pgd = link->parent->gd;
309 
310 		if (pgd)
311 			pgd->max_off_time_changed = true;
312 	}
313 
314 	gd->max_off_time_ns = -1;
315 	gd->max_off_time_changed = false;
316 	gd->cached_power_down_ok = true;
317 
318 	/*
319 	 * Find a state to power down to, starting from the state
320 	 * determined by the next wakeup.
321 	 */
322 	while (!__default_power_down_ok(pd, state_idx)) {
323 		if (state_idx == 0) {
324 			gd->cached_power_down_ok = false;
325 			break;
326 		}
327 		state_idx--;
328 	}
329 
330 done:
331 	genpd->state_idx = state_idx;
332 	gd->cached_power_down_state_idx = genpd->state_idx;
333 	return gd->cached_power_down_ok;
334 }
335 
336 static bool default_power_down_ok(struct dev_pm_domain *pd)
337 {
338 	return _default_power_down_ok(pd, ktime_get());
339 }
340 
341 #ifdef CONFIG_CPU_IDLE
342 static bool cpu_power_down_ok(struct dev_pm_domain *pd)
343 {
344 	struct generic_pm_domain *genpd = pd_to_genpd(pd);
345 	struct cpuidle_device *dev;
346 	ktime_t domain_wakeup, next_hrtimer;
347 	ktime_t now = ktime_get();
348 	s64 idle_duration_ns;
349 	int cpu, i;
350 
351 	/* Validate dev PM QoS constraints. */
352 	if (!_default_power_down_ok(pd, now))
353 		return false;
354 
355 	if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN))
356 		return true;
357 
358 	/*
359 	 * Find the next wakeup for any of the online CPUs within the PM domain
360 	 * and its subdomains. Note, we only need the genpd->cpus, as it already
361 	 * contains a mask of all CPUs from subdomains.
362 	 */
363 	domain_wakeup = ktime_set(KTIME_SEC_MAX, 0);
364 	for_each_cpu_and(cpu, genpd->cpus, cpu_online_mask) {
365 		dev = per_cpu(cpuidle_devices, cpu);
366 		if (dev) {
367 			next_hrtimer = READ_ONCE(dev->next_hrtimer);
368 			if (ktime_before(next_hrtimer, domain_wakeup))
369 				domain_wakeup = next_hrtimer;
370 		}
371 	}
372 
373 	/* The minimum idle duration is from now - until the next wakeup. */
374 	idle_duration_ns = ktime_to_ns(ktime_sub(domain_wakeup, now));
375 	if (idle_duration_ns <= 0)
376 		return false;
377 
378 	/* Store the next domain_wakeup to allow consumers to use it. */
379 	genpd->gd->next_hrtimer = domain_wakeup;
380 
381 	/*
382 	 * Find the deepest idle state that has its residency value satisfied
383 	 * and by also taking into account the power off latency for the state.
384 	 * Start at the state picked by the dev PM QoS constraint validation.
385 	 */
386 	i = genpd->state_idx;
387 	do {
388 		if (idle_duration_ns >= (genpd->states[i].residency_ns +
389 		    genpd->states[i].power_off_latency_ns)) {
390 			genpd->state_idx = i;
391 			return true;
392 		}
393 	} while (--i >= 0);
394 
395 	return false;
396 }
397 
398 struct dev_power_governor pm_domain_cpu_gov = {
399 	.suspend_ok = default_suspend_ok,
400 	.power_down_ok = cpu_power_down_ok,
401 };
402 #endif
403 
404 struct dev_power_governor simple_qos_governor = {
405 	.suspend_ok = default_suspend_ok,
406 	.power_down_ok = default_power_down_ok,
407 };
408 
409 /**
410  * pm_genpd_gov_always_on - A governor implementing an always-on policy
411  */
412 struct dev_power_governor pm_domain_always_on_gov = {
413 	.suspend_ok = default_suspend_ok,
414 };
415