xref: /openbmc/linux/drivers/base/power/runtime.c (revision ee7da21a)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * drivers/base/power/runtime.c - Helper functions for device runtime PM
4  *
5  * Copyright (c) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc.
6  * Copyright (C) 2010 Alan Stern <stern@rowland.harvard.edu>
7  */
8 #include <linux/sched/mm.h>
9 #include <linux/ktime.h>
10 #include <linux/hrtimer.h>
11 #include <linux/export.h>
12 #include <linux/pm_runtime.h>
13 #include <linux/pm_wakeirq.h>
14 #include <trace/events/rpm.h>
15 
16 #include "../base.h"
17 #include "power.h"
18 
19 typedef int (*pm_callback_t)(struct device *);
20 
21 static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset)
22 {
23 	pm_callback_t cb;
24 	const struct dev_pm_ops *ops;
25 
26 	if (dev->pm_domain)
27 		ops = &dev->pm_domain->ops;
28 	else if (dev->type && dev->type->pm)
29 		ops = dev->type->pm;
30 	else if (dev->class && dev->class->pm)
31 		ops = dev->class->pm;
32 	else if (dev->bus && dev->bus->pm)
33 		ops = dev->bus->pm;
34 	else
35 		ops = NULL;
36 
37 	if (ops)
38 		cb = *(pm_callback_t *)((void *)ops + cb_offset);
39 	else
40 		cb = NULL;
41 
42 	if (!cb && dev->driver && dev->driver->pm)
43 		cb = *(pm_callback_t *)((void *)dev->driver->pm + cb_offset);
44 
45 	return cb;
46 }
47 
48 #define RPM_GET_CALLBACK(dev, callback) \
49 		__rpm_get_callback(dev, offsetof(struct dev_pm_ops, callback))
50 
51 static int rpm_resume(struct device *dev, int rpmflags);
52 static int rpm_suspend(struct device *dev, int rpmflags);
53 
54 /**
55  * update_pm_runtime_accounting - Update the time accounting of power states
56  * @dev: Device to update the accounting for
57  *
58  * In order to be able to have time accounting of the various power states
59  * (as used by programs such as PowerTOP to show the effectiveness of runtime
60  * PM), we need to track the time spent in each state.
61  * update_pm_runtime_accounting must be called each time before the
62  * runtime_status field is updated, to account the time in the old state
63  * correctly.
64  */
65 static void update_pm_runtime_accounting(struct device *dev)
66 {
67 	u64 now, last, delta;
68 
69 	if (dev->power.disable_depth > 0)
70 		return;
71 
72 	last = dev->power.accounting_timestamp;
73 
74 	now = ktime_get_mono_fast_ns();
75 	dev->power.accounting_timestamp = now;
76 
77 	/*
78 	 * Because ktime_get_mono_fast_ns() is not monotonic during
79 	 * timekeeping updates, ensure that 'now' is after the last saved
80 	 * timesptamp.
81 	 */
82 	if (now < last)
83 		return;
84 
85 	delta = now - last;
86 
87 	if (dev->power.runtime_status == RPM_SUSPENDED)
88 		dev->power.suspended_time += delta;
89 	else
90 		dev->power.active_time += delta;
91 }
92 
93 static void __update_runtime_status(struct device *dev, enum rpm_status status)
94 {
95 	update_pm_runtime_accounting(dev);
96 	dev->power.runtime_status = status;
97 }
98 
99 static u64 rpm_get_accounted_time(struct device *dev, bool suspended)
100 {
101 	u64 time;
102 	unsigned long flags;
103 
104 	spin_lock_irqsave(&dev->power.lock, flags);
105 
106 	update_pm_runtime_accounting(dev);
107 	time = suspended ? dev->power.suspended_time : dev->power.active_time;
108 
109 	spin_unlock_irqrestore(&dev->power.lock, flags);
110 
111 	return time;
112 }
113 
114 u64 pm_runtime_active_time(struct device *dev)
115 {
116 	return rpm_get_accounted_time(dev, false);
117 }
118 
119 u64 pm_runtime_suspended_time(struct device *dev)
120 {
121 	return rpm_get_accounted_time(dev, true);
122 }
123 EXPORT_SYMBOL_GPL(pm_runtime_suspended_time);
124 
125 /**
126  * pm_runtime_deactivate_timer - Deactivate given device's suspend timer.
127  * @dev: Device to handle.
128  */
129 static void pm_runtime_deactivate_timer(struct device *dev)
130 {
131 	if (dev->power.timer_expires > 0) {
132 		hrtimer_try_to_cancel(&dev->power.suspend_timer);
133 		dev->power.timer_expires = 0;
134 	}
135 }
136 
137 /**
138  * pm_runtime_cancel_pending - Deactivate suspend timer and cancel requests.
139  * @dev: Device to handle.
140  */
141 static void pm_runtime_cancel_pending(struct device *dev)
142 {
143 	pm_runtime_deactivate_timer(dev);
144 	/*
145 	 * In case there's a request pending, make sure its work function will
146 	 * return without doing anything.
147 	 */
148 	dev->power.request = RPM_REQ_NONE;
149 }
150 
151 /*
152  * pm_runtime_autosuspend_expiration - Get a device's autosuspend-delay expiration time.
153  * @dev: Device to handle.
154  *
155  * Compute the autosuspend-delay expiration time based on the device's
156  * power.last_busy time.  If the delay has already expired or is disabled
157  * (negative) or the power.use_autosuspend flag isn't set, return 0.
158  * Otherwise return the expiration time in nanoseconds (adjusted to be nonzero).
159  *
160  * This function may be called either with or without dev->power.lock held.
161  * Either way it can be racy, since power.last_busy may be updated at any time.
162  */
163 u64 pm_runtime_autosuspend_expiration(struct device *dev)
164 {
165 	int autosuspend_delay;
166 	u64 expires;
167 
168 	if (!dev->power.use_autosuspend)
169 		return 0;
170 
171 	autosuspend_delay = READ_ONCE(dev->power.autosuspend_delay);
172 	if (autosuspend_delay < 0)
173 		return 0;
174 
175 	expires  = READ_ONCE(dev->power.last_busy);
176 	expires += (u64)autosuspend_delay * NSEC_PER_MSEC;
177 	if (expires > ktime_get_mono_fast_ns())
178 		return expires;	/* Expires in the future */
179 
180 	return 0;
181 }
182 EXPORT_SYMBOL_GPL(pm_runtime_autosuspend_expiration);
183 
184 static int dev_memalloc_noio(struct device *dev, void *data)
185 {
186 	return dev->power.memalloc_noio;
187 }
188 
189 /*
190  * pm_runtime_set_memalloc_noio - Set a device's memalloc_noio flag.
191  * @dev: Device to handle.
192  * @enable: True for setting the flag and False for clearing the flag.
193  *
194  * Set the flag for all devices in the path from the device to the
195  * root device in the device tree if @enable is true, otherwise clear
196  * the flag for devices in the path whose siblings don't set the flag.
197  *
198  * The function should only be called by block device, or network
199  * device driver for solving the deadlock problem during runtime
200  * resume/suspend:
201  *
202  *     If memory allocation with GFP_KERNEL is called inside runtime
203  *     resume/suspend callback of any one of its ancestors(or the
204  *     block device itself), the deadlock may be triggered inside the
205  *     memory allocation since it might not complete until the block
206  *     device becomes active and the involed page I/O finishes. The
207  *     situation is pointed out first by Alan Stern. Network device
208  *     are involved in iSCSI kind of situation.
209  *
210  * The lock of dev_hotplug_mutex is held in the function for handling
211  * hotplug race because pm_runtime_set_memalloc_noio() may be called
212  * in async probe().
213  *
214  * The function should be called between device_add() and device_del()
215  * on the affected device(block/network device).
216  */
217 void pm_runtime_set_memalloc_noio(struct device *dev, bool enable)
218 {
219 	static DEFINE_MUTEX(dev_hotplug_mutex);
220 
221 	mutex_lock(&dev_hotplug_mutex);
222 	for (;;) {
223 		bool enabled;
224 
225 		/* hold power lock since bitfield is not SMP-safe. */
226 		spin_lock_irq(&dev->power.lock);
227 		enabled = dev->power.memalloc_noio;
228 		dev->power.memalloc_noio = enable;
229 		spin_unlock_irq(&dev->power.lock);
230 
231 		/*
232 		 * not need to enable ancestors any more if the device
233 		 * has been enabled.
234 		 */
235 		if (enabled && enable)
236 			break;
237 
238 		dev = dev->parent;
239 
240 		/*
241 		 * clear flag of the parent device only if all the
242 		 * children don't set the flag because ancestor's
243 		 * flag was set by any one of the descendants.
244 		 */
245 		if (!dev || (!enable &&
246 			     device_for_each_child(dev, NULL,
247 						   dev_memalloc_noio)))
248 			break;
249 	}
250 	mutex_unlock(&dev_hotplug_mutex);
251 }
252 EXPORT_SYMBOL_GPL(pm_runtime_set_memalloc_noio);
253 
254 /**
255  * rpm_check_suspend_allowed - Test whether a device may be suspended.
256  * @dev: Device to test.
257  */
258 static int rpm_check_suspend_allowed(struct device *dev)
259 {
260 	int retval = 0;
261 
262 	if (dev->power.runtime_error)
263 		retval = -EINVAL;
264 	else if (dev->power.disable_depth > 0)
265 		retval = -EACCES;
266 	else if (atomic_read(&dev->power.usage_count) > 0)
267 		retval = -EAGAIN;
268 	else if (!dev->power.ignore_children &&
269 			atomic_read(&dev->power.child_count))
270 		retval = -EBUSY;
271 
272 	/* Pending resume requests take precedence over suspends. */
273 	else if ((dev->power.deferred_resume
274 			&& dev->power.runtime_status == RPM_SUSPENDING)
275 	    || (dev->power.request_pending
276 			&& dev->power.request == RPM_REQ_RESUME))
277 		retval = -EAGAIN;
278 	else if (__dev_pm_qos_resume_latency(dev) == 0)
279 		retval = -EPERM;
280 	else if (dev->power.runtime_status == RPM_SUSPENDED)
281 		retval = 1;
282 
283 	return retval;
284 }
285 
286 static int rpm_get_suppliers(struct device *dev)
287 {
288 	struct device_link *link;
289 
290 	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
291 				device_links_read_lock_held()) {
292 		int retval;
293 
294 		if (!(link->flags & DL_FLAG_PM_RUNTIME))
295 			continue;
296 
297 		retval = pm_runtime_get_sync(link->supplier);
298 		/* Ignore suppliers with disabled runtime PM. */
299 		if (retval < 0 && retval != -EACCES) {
300 			pm_runtime_put_noidle(link->supplier);
301 			return retval;
302 		}
303 		refcount_inc(&link->rpm_active);
304 	}
305 	return 0;
306 }
307 
308 static void __rpm_put_suppliers(struct device *dev, bool try_to_suspend)
309 {
310 	struct device_link *link;
311 
312 	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
313 				device_links_read_lock_held()) {
314 
315 		while (refcount_dec_not_one(&link->rpm_active))
316 			pm_runtime_put_noidle(link->supplier);
317 
318 		if (try_to_suspend)
319 			pm_request_idle(link->supplier);
320 	}
321 }
322 
323 static void rpm_put_suppliers(struct device *dev)
324 {
325 	__rpm_put_suppliers(dev, true);
326 }
327 
328 static void rpm_suspend_suppliers(struct device *dev)
329 {
330 	struct device_link *link;
331 	int idx = device_links_read_lock();
332 
333 	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
334 				device_links_read_lock_held())
335 		pm_request_idle(link->supplier);
336 
337 	device_links_read_unlock(idx);
338 }
339 
340 /**
341  * __rpm_callback - Run a given runtime PM callback for a given device.
342  * @cb: Runtime PM callback to run.
343  * @dev: Device to run the callback for.
344  */
345 static int __rpm_callback(int (*cb)(struct device *), struct device *dev)
346 	__releases(&dev->power.lock) __acquires(&dev->power.lock)
347 {
348 	int retval = 0, idx;
349 	bool use_links = dev->power.links_count > 0;
350 
351 	if (dev->power.irq_safe) {
352 		spin_unlock(&dev->power.lock);
353 	} else {
354 		spin_unlock_irq(&dev->power.lock);
355 
356 		/*
357 		 * Resume suppliers if necessary.
358 		 *
359 		 * The device's runtime PM status cannot change until this
360 		 * routine returns, so it is safe to read the status outside of
361 		 * the lock.
362 		 */
363 		if (use_links && dev->power.runtime_status == RPM_RESUMING) {
364 			idx = device_links_read_lock();
365 
366 			retval = rpm_get_suppliers(dev);
367 			if (retval) {
368 				rpm_put_suppliers(dev);
369 				goto fail;
370 			}
371 
372 			device_links_read_unlock(idx);
373 		}
374 	}
375 
376 	if (cb)
377 		retval = cb(dev);
378 
379 	if (dev->power.irq_safe) {
380 		spin_lock(&dev->power.lock);
381 	} else {
382 		/*
383 		 * If the device is suspending and the callback has returned
384 		 * success, drop the usage counters of the suppliers that have
385 		 * been reference counted on its resume.
386 		 *
387 		 * Do that if resume fails too.
388 		 */
389 		if (use_links
390 		    && ((dev->power.runtime_status == RPM_SUSPENDING && !retval)
391 		    || (dev->power.runtime_status == RPM_RESUMING && retval))) {
392 			idx = device_links_read_lock();
393 
394 			__rpm_put_suppliers(dev, false);
395 
396 fail:
397 			device_links_read_unlock(idx);
398 		}
399 
400 		spin_lock_irq(&dev->power.lock);
401 	}
402 
403 	return retval;
404 }
405 
406 /**
407  * rpm_idle - Notify device bus type if the device can be suspended.
408  * @dev: Device to notify the bus type about.
409  * @rpmflags: Flag bits.
410  *
411  * Check if the device's runtime PM status allows it to be suspended.  If
412  * another idle notification has been started earlier, return immediately.  If
413  * the RPM_ASYNC flag is set then queue an idle-notification request; otherwise
414  * run the ->runtime_idle() callback directly. If the ->runtime_idle callback
415  * doesn't exist or if it returns 0, call rpm_suspend with the RPM_AUTO flag.
416  *
417  * This function must be called under dev->power.lock with interrupts disabled.
418  */
419 static int rpm_idle(struct device *dev, int rpmflags)
420 {
421 	int (*callback)(struct device *);
422 	int retval;
423 
424 	trace_rpm_idle_rcuidle(dev, rpmflags);
425 	retval = rpm_check_suspend_allowed(dev);
426 	if (retval < 0)
427 		;	/* Conditions are wrong. */
428 
429 	/* Idle notifications are allowed only in the RPM_ACTIVE state. */
430 	else if (dev->power.runtime_status != RPM_ACTIVE)
431 		retval = -EAGAIN;
432 
433 	/*
434 	 * Any pending request other than an idle notification takes
435 	 * precedence over us, except that the timer may be running.
436 	 */
437 	else if (dev->power.request_pending &&
438 	    dev->power.request > RPM_REQ_IDLE)
439 		retval = -EAGAIN;
440 
441 	/* Act as though RPM_NOWAIT is always set. */
442 	else if (dev->power.idle_notification)
443 		retval = -EINPROGRESS;
444 	if (retval)
445 		goto out;
446 
447 	/* Pending requests need to be canceled. */
448 	dev->power.request = RPM_REQ_NONE;
449 
450 	callback = RPM_GET_CALLBACK(dev, runtime_idle);
451 
452 	/* If no callback assume success. */
453 	if (!callback || dev->power.no_callbacks)
454 		goto out;
455 
456 	/* Carry out an asynchronous or a synchronous idle notification. */
457 	if (rpmflags & RPM_ASYNC) {
458 		dev->power.request = RPM_REQ_IDLE;
459 		if (!dev->power.request_pending) {
460 			dev->power.request_pending = true;
461 			queue_work(pm_wq, &dev->power.work);
462 		}
463 		trace_rpm_return_int_rcuidle(dev, _THIS_IP_, 0);
464 		return 0;
465 	}
466 
467 	dev->power.idle_notification = true;
468 
469 	retval = __rpm_callback(callback, dev);
470 
471 	dev->power.idle_notification = false;
472 	wake_up_all(&dev->power.wait_queue);
473 
474  out:
475 	trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
476 	return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO);
477 }
478 
479 /**
480  * rpm_callback - Run a given runtime PM callback for a given device.
481  * @cb: Runtime PM callback to run.
482  * @dev: Device to run the callback for.
483  */
484 static int rpm_callback(int (*cb)(struct device *), struct device *dev)
485 {
486 	int retval;
487 
488 	if (dev->power.memalloc_noio) {
489 		unsigned int noio_flag;
490 
491 		/*
492 		 * Deadlock might be caused if memory allocation with
493 		 * GFP_KERNEL happens inside runtime_suspend and
494 		 * runtime_resume callbacks of one block device's
495 		 * ancestor or the block device itself. Network
496 		 * device might be thought as part of iSCSI block
497 		 * device, so network device and its ancestor should
498 		 * be marked as memalloc_noio too.
499 		 */
500 		noio_flag = memalloc_noio_save();
501 		retval = __rpm_callback(cb, dev);
502 		memalloc_noio_restore(noio_flag);
503 	} else {
504 		retval = __rpm_callback(cb, dev);
505 	}
506 
507 	dev->power.runtime_error = retval;
508 	return retval != -EACCES ? retval : -EIO;
509 }
510 
511 /**
512  * rpm_suspend - Carry out runtime suspend of given device.
513  * @dev: Device to suspend.
514  * @rpmflags: Flag bits.
515  *
516  * Check if the device's runtime PM status allows it to be suspended.
517  * Cancel a pending idle notification, autosuspend or suspend. If
518  * another suspend has been started earlier, either return immediately
519  * or wait for it to finish, depending on the RPM_NOWAIT and RPM_ASYNC
520  * flags. If the RPM_ASYNC flag is set then queue a suspend request;
521  * otherwise run the ->runtime_suspend() callback directly. When
522  * ->runtime_suspend succeeded, if a deferred resume was requested while
523  * the callback was running then carry it out, otherwise send an idle
524  * notification for its parent (if the suspend succeeded and both
525  * ignore_children of parent->power and irq_safe of dev->power are not set).
526  * If ->runtime_suspend failed with -EAGAIN or -EBUSY, and if the RPM_AUTO
527  * flag is set and the next autosuspend-delay expiration time is in the
528  * future, schedule another autosuspend attempt.
529  *
530  * This function must be called under dev->power.lock with interrupts disabled.
531  */
532 static int rpm_suspend(struct device *dev, int rpmflags)
533 	__releases(&dev->power.lock) __acquires(&dev->power.lock)
534 {
535 	int (*callback)(struct device *);
536 	struct device *parent = NULL;
537 	int retval;
538 
539 	trace_rpm_suspend_rcuidle(dev, rpmflags);
540 
541  repeat:
542 	retval = rpm_check_suspend_allowed(dev);
543 	if (retval < 0)
544 		goto out;	/* Conditions are wrong. */
545 
546 	/* Synchronous suspends are not allowed in the RPM_RESUMING state. */
547 	if (dev->power.runtime_status == RPM_RESUMING && !(rpmflags & RPM_ASYNC))
548 		retval = -EAGAIN;
549 	if (retval)
550 		goto out;
551 
552 	/* If the autosuspend_delay time hasn't expired yet, reschedule. */
553 	if ((rpmflags & RPM_AUTO)
554 	    && dev->power.runtime_status != RPM_SUSPENDING) {
555 		u64 expires = pm_runtime_autosuspend_expiration(dev);
556 
557 		if (expires != 0) {
558 			/* Pending requests need to be canceled. */
559 			dev->power.request = RPM_REQ_NONE;
560 
561 			/*
562 			 * Optimization: If the timer is already running and is
563 			 * set to expire at or before the autosuspend delay,
564 			 * avoid the overhead of resetting it.  Just let it
565 			 * expire; pm_suspend_timer_fn() will take care of the
566 			 * rest.
567 			 */
568 			if (!(dev->power.timer_expires &&
569 					dev->power.timer_expires <= expires)) {
570 				/*
571 				 * We add a slack of 25% to gather wakeups
572 				 * without sacrificing the granularity.
573 				 */
574 				u64 slack = (u64)READ_ONCE(dev->power.autosuspend_delay) *
575 						    (NSEC_PER_MSEC >> 2);
576 
577 				dev->power.timer_expires = expires;
578 				hrtimer_start_range_ns(&dev->power.suspend_timer,
579 						ns_to_ktime(expires),
580 						slack,
581 						HRTIMER_MODE_ABS);
582 			}
583 			dev->power.timer_autosuspends = 1;
584 			goto out;
585 		}
586 	}
587 
588 	/* Other scheduled or pending requests need to be canceled. */
589 	pm_runtime_cancel_pending(dev);
590 
591 	if (dev->power.runtime_status == RPM_SUSPENDING) {
592 		DEFINE_WAIT(wait);
593 
594 		if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
595 			retval = -EINPROGRESS;
596 			goto out;
597 		}
598 
599 		if (dev->power.irq_safe) {
600 			spin_unlock(&dev->power.lock);
601 
602 			cpu_relax();
603 
604 			spin_lock(&dev->power.lock);
605 			goto repeat;
606 		}
607 
608 		/* Wait for the other suspend running in parallel with us. */
609 		for (;;) {
610 			prepare_to_wait(&dev->power.wait_queue, &wait,
611 					TASK_UNINTERRUPTIBLE);
612 			if (dev->power.runtime_status != RPM_SUSPENDING)
613 				break;
614 
615 			spin_unlock_irq(&dev->power.lock);
616 
617 			schedule();
618 
619 			spin_lock_irq(&dev->power.lock);
620 		}
621 		finish_wait(&dev->power.wait_queue, &wait);
622 		goto repeat;
623 	}
624 
625 	if (dev->power.no_callbacks)
626 		goto no_callback;	/* Assume success. */
627 
628 	/* Carry out an asynchronous or a synchronous suspend. */
629 	if (rpmflags & RPM_ASYNC) {
630 		dev->power.request = (rpmflags & RPM_AUTO) ?
631 		    RPM_REQ_AUTOSUSPEND : RPM_REQ_SUSPEND;
632 		if (!dev->power.request_pending) {
633 			dev->power.request_pending = true;
634 			queue_work(pm_wq, &dev->power.work);
635 		}
636 		goto out;
637 	}
638 
639 	__update_runtime_status(dev, RPM_SUSPENDING);
640 
641 	callback = RPM_GET_CALLBACK(dev, runtime_suspend);
642 
643 	dev_pm_enable_wake_irq_check(dev, true);
644 	retval = rpm_callback(callback, dev);
645 	if (retval)
646 		goto fail;
647 
648  no_callback:
649 	__update_runtime_status(dev, RPM_SUSPENDED);
650 	pm_runtime_deactivate_timer(dev);
651 
652 	if (dev->parent) {
653 		parent = dev->parent;
654 		atomic_add_unless(&parent->power.child_count, -1, 0);
655 	}
656 	wake_up_all(&dev->power.wait_queue);
657 
658 	if (dev->power.deferred_resume) {
659 		dev->power.deferred_resume = false;
660 		rpm_resume(dev, 0);
661 		retval = -EAGAIN;
662 		goto out;
663 	}
664 
665 	if (dev->power.irq_safe)
666 		goto out;
667 
668 	/* Maybe the parent is now able to suspend. */
669 	if (parent && !parent->power.ignore_children) {
670 		spin_unlock(&dev->power.lock);
671 
672 		spin_lock(&parent->power.lock);
673 		rpm_idle(parent, RPM_ASYNC);
674 		spin_unlock(&parent->power.lock);
675 
676 		spin_lock(&dev->power.lock);
677 	}
678 	/* Maybe the suppliers are now able to suspend. */
679 	if (dev->power.links_count > 0) {
680 		spin_unlock_irq(&dev->power.lock);
681 
682 		rpm_suspend_suppliers(dev);
683 
684 		spin_lock_irq(&dev->power.lock);
685 	}
686 
687  out:
688 	trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
689 
690 	return retval;
691 
692  fail:
693 	dev_pm_disable_wake_irq_check(dev);
694 	__update_runtime_status(dev, RPM_ACTIVE);
695 	dev->power.deferred_resume = false;
696 	wake_up_all(&dev->power.wait_queue);
697 
698 	if (retval == -EAGAIN || retval == -EBUSY) {
699 		dev->power.runtime_error = 0;
700 
701 		/*
702 		 * If the callback routine failed an autosuspend, and
703 		 * if the last_busy time has been updated so that there
704 		 * is a new autosuspend expiration time, automatically
705 		 * reschedule another autosuspend.
706 		 */
707 		if ((rpmflags & RPM_AUTO) &&
708 		    pm_runtime_autosuspend_expiration(dev) != 0)
709 			goto repeat;
710 	} else {
711 		pm_runtime_cancel_pending(dev);
712 	}
713 	goto out;
714 }
715 
716 /**
717  * rpm_resume - Carry out runtime resume of given device.
718  * @dev: Device to resume.
719  * @rpmflags: Flag bits.
720  *
721  * Check if the device's runtime PM status allows it to be resumed.  Cancel
722  * any scheduled or pending requests.  If another resume has been started
723  * earlier, either return immediately or wait for it to finish, depending on the
724  * RPM_NOWAIT and RPM_ASYNC flags.  Similarly, if there's a suspend running in
725  * parallel with this function, either tell the other process to resume after
726  * suspending (deferred_resume) or wait for it to finish.  If the RPM_ASYNC
727  * flag is set then queue a resume request; otherwise run the
728  * ->runtime_resume() callback directly.  Queue an idle notification for the
729  * device if the resume succeeded.
730  *
731  * This function must be called under dev->power.lock with interrupts disabled.
732  */
733 static int rpm_resume(struct device *dev, int rpmflags)
734 	__releases(&dev->power.lock) __acquires(&dev->power.lock)
735 {
736 	int (*callback)(struct device *);
737 	struct device *parent = NULL;
738 	int retval = 0;
739 
740 	trace_rpm_resume_rcuidle(dev, rpmflags);
741 
742  repeat:
743 	if (dev->power.runtime_error)
744 		retval = -EINVAL;
745 	else if (dev->power.disable_depth == 1 && dev->power.is_suspended
746 	    && dev->power.runtime_status == RPM_ACTIVE)
747 		retval = 1;
748 	else if (dev->power.disable_depth > 0)
749 		retval = -EACCES;
750 	if (retval)
751 		goto out;
752 
753 	/*
754 	 * Other scheduled or pending requests need to be canceled.  Small
755 	 * optimization: If an autosuspend timer is running, leave it running
756 	 * rather than cancelling it now only to restart it again in the near
757 	 * future.
758 	 */
759 	dev->power.request = RPM_REQ_NONE;
760 	if (!dev->power.timer_autosuspends)
761 		pm_runtime_deactivate_timer(dev);
762 
763 	if (dev->power.runtime_status == RPM_ACTIVE) {
764 		retval = 1;
765 		goto out;
766 	}
767 
768 	if (dev->power.runtime_status == RPM_RESUMING
769 	    || dev->power.runtime_status == RPM_SUSPENDING) {
770 		DEFINE_WAIT(wait);
771 
772 		if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
773 			if (dev->power.runtime_status == RPM_SUSPENDING)
774 				dev->power.deferred_resume = true;
775 			else
776 				retval = -EINPROGRESS;
777 			goto out;
778 		}
779 
780 		if (dev->power.irq_safe) {
781 			spin_unlock(&dev->power.lock);
782 
783 			cpu_relax();
784 
785 			spin_lock(&dev->power.lock);
786 			goto repeat;
787 		}
788 
789 		/* Wait for the operation carried out in parallel with us. */
790 		for (;;) {
791 			prepare_to_wait(&dev->power.wait_queue, &wait,
792 					TASK_UNINTERRUPTIBLE);
793 			if (dev->power.runtime_status != RPM_RESUMING
794 			    && dev->power.runtime_status != RPM_SUSPENDING)
795 				break;
796 
797 			spin_unlock_irq(&dev->power.lock);
798 
799 			schedule();
800 
801 			spin_lock_irq(&dev->power.lock);
802 		}
803 		finish_wait(&dev->power.wait_queue, &wait);
804 		goto repeat;
805 	}
806 
807 	/*
808 	 * See if we can skip waking up the parent.  This is safe only if
809 	 * power.no_callbacks is set, because otherwise we don't know whether
810 	 * the resume will actually succeed.
811 	 */
812 	if (dev->power.no_callbacks && !parent && dev->parent) {
813 		spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING);
814 		if (dev->parent->power.disable_depth > 0
815 		    || dev->parent->power.ignore_children
816 		    || dev->parent->power.runtime_status == RPM_ACTIVE) {
817 			atomic_inc(&dev->parent->power.child_count);
818 			spin_unlock(&dev->parent->power.lock);
819 			retval = 1;
820 			goto no_callback;	/* Assume success. */
821 		}
822 		spin_unlock(&dev->parent->power.lock);
823 	}
824 
825 	/* Carry out an asynchronous or a synchronous resume. */
826 	if (rpmflags & RPM_ASYNC) {
827 		dev->power.request = RPM_REQ_RESUME;
828 		if (!dev->power.request_pending) {
829 			dev->power.request_pending = true;
830 			queue_work(pm_wq, &dev->power.work);
831 		}
832 		retval = 0;
833 		goto out;
834 	}
835 
836 	if (!parent && dev->parent) {
837 		/*
838 		 * Increment the parent's usage counter and resume it if
839 		 * necessary.  Not needed if dev is irq-safe; then the
840 		 * parent is permanently resumed.
841 		 */
842 		parent = dev->parent;
843 		if (dev->power.irq_safe)
844 			goto skip_parent;
845 		spin_unlock(&dev->power.lock);
846 
847 		pm_runtime_get_noresume(parent);
848 
849 		spin_lock(&parent->power.lock);
850 		/*
851 		 * Resume the parent if it has runtime PM enabled and not been
852 		 * set to ignore its children.
853 		 */
854 		if (!parent->power.disable_depth
855 		    && !parent->power.ignore_children) {
856 			rpm_resume(parent, 0);
857 			if (parent->power.runtime_status != RPM_ACTIVE)
858 				retval = -EBUSY;
859 		}
860 		spin_unlock(&parent->power.lock);
861 
862 		spin_lock(&dev->power.lock);
863 		if (retval)
864 			goto out;
865 		goto repeat;
866 	}
867  skip_parent:
868 
869 	if (dev->power.no_callbacks)
870 		goto no_callback;	/* Assume success. */
871 
872 	__update_runtime_status(dev, RPM_RESUMING);
873 
874 	callback = RPM_GET_CALLBACK(dev, runtime_resume);
875 
876 	dev_pm_disable_wake_irq_check(dev);
877 	retval = rpm_callback(callback, dev);
878 	if (retval) {
879 		__update_runtime_status(dev, RPM_SUSPENDED);
880 		pm_runtime_cancel_pending(dev);
881 		dev_pm_enable_wake_irq_check(dev, false);
882 	} else {
883  no_callback:
884 		__update_runtime_status(dev, RPM_ACTIVE);
885 		pm_runtime_mark_last_busy(dev);
886 		if (parent)
887 			atomic_inc(&parent->power.child_count);
888 	}
889 	wake_up_all(&dev->power.wait_queue);
890 
891 	if (retval >= 0)
892 		rpm_idle(dev, RPM_ASYNC);
893 
894  out:
895 	if (parent && !dev->power.irq_safe) {
896 		spin_unlock_irq(&dev->power.lock);
897 
898 		pm_runtime_put(parent);
899 
900 		spin_lock_irq(&dev->power.lock);
901 	}
902 
903 	trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
904 
905 	return retval;
906 }
907 
908 /**
909  * pm_runtime_work - Universal runtime PM work function.
910  * @work: Work structure used for scheduling the execution of this function.
911  *
912  * Use @work to get the device object the work is to be done for, determine what
913  * is to be done and execute the appropriate runtime PM function.
914  */
915 static void pm_runtime_work(struct work_struct *work)
916 {
917 	struct device *dev = container_of(work, struct device, power.work);
918 	enum rpm_request req;
919 
920 	spin_lock_irq(&dev->power.lock);
921 
922 	if (!dev->power.request_pending)
923 		goto out;
924 
925 	req = dev->power.request;
926 	dev->power.request = RPM_REQ_NONE;
927 	dev->power.request_pending = false;
928 
929 	switch (req) {
930 	case RPM_REQ_NONE:
931 		break;
932 	case RPM_REQ_IDLE:
933 		rpm_idle(dev, RPM_NOWAIT);
934 		break;
935 	case RPM_REQ_SUSPEND:
936 		rpm_suspend(dev, RPM_NOWAIT);
937 		break;
938 	case RPM_REQ_AUTOSUSPEND:
939 		rpm_suspend(dev, RPM_NOWAIT | RPM_AUTO);
940 		break;
941 	case RPM_REQ_RESUME:
942 		rpm_resume(dev, RPM_NOWAIT);
943 		break;
944 	}
945 
946  out:
947 	spin_unlock_irq(&dev->power.lock);
948 }
949 
950 /**
951  * pm_suspend_timer_fn - Timer function for pm_schedule_suspend().
952  * @timer: hrtimer used by pm_schedule_suspend().
953  *
954  * Check if the time is right and queue a suspend request.
955  */
956 static enum hrtimer_restart  pm_suspend_timer_fn(struct hrtimer *timer)
957 {
958 	struct device *dev = container_of(timer, struct device, power.suspend_timer);
959 	unsigned long flags;
960 	u64 expires;
961 
962 	spin_lock_irqsave(&dev->power.lock, flags);
963 
964 	expires = dev->power.timer_expires;
965 	/*
966 	 * If 'expires' is after the current time, we've been called
967 	 * too early.
968 	 */
969 	if (expires > 0 && expires < ktime_get_mono_fast_ns()) {
970 		dev->power.timer_expires = 0;
971 		rpm_suspend(dev, dev->power.timer_autosuspends ?
972 		    (RPM_ASYNC | RPM_AUTO) : RPM_ASYNC);
973 	}
974 
975 	spin_unlock_irqrestore(&dev->power.lock, flags);
976 
977 	return HRTIMER_NORESTART;
978 }
979 
980 /**
981  * pm_schedule_suspend - Set up a timer to submit a suspend request in future.
982  * @dev: Device to suspend.
983  * @delay: Time to wait before submitting a suspend request, in milliseconds.
984  */
985 int pm_schedule_suspend(struct device *dev, unsigned int delay)
986 {
987 	unsigned long flags;
988 	u64 expires;
989 	int retval;
990 
991 	spin_lock_irqsave(&dev->power.lock, flags);
992 
993 	if (!delay) {
994 		retval = rpm_suspend(dev, RPM_ASYNC);
995 		goto out;
996 	}
997 
998 	retval = rpm_check_suspend_allowed(dev);
999 	if (retval)
1000 		goto out;
1001 
1002 	/* Other scheduled or pending requests need to be canceled. */
1003 	pm_runtime_cancel_pending(dev);
1004 
1005 	expires = ktime_get_mono_fast_ns() + (u64)delay * NSEC_PER_MSEC;
1006 	dev->power.timer_expires = expires;
1007 	dev->power.timer_autosuspends = 0;
1008 	hrtimer_start(&dev->power.suspend_timer, expires, HRTIMER_MODE_ABS);
1009 
1010  out:
1011 	spin_unlock_irqrestore(&dev->power.lock, flags);
1012 
1013 	return retval;
1014 }
1015 EXPORT_SYMBOL_GPL(pm_schedule_suspend);
1016 
1017 /**
1018  * __pm_runtime_idle - Entry point for runtime idle operations.
1019  * @dev: Device to send idle notification for.
1020  * @rpmflags: Flag bits.
1021  *
1022  * If the RPM_GET_PUT flag is set, decrement the device's usage count and
1023  * return immediately if it is larger than zero.  Then carry out an idle
1024  * notification, either synchronous or asynchronous.
1025  *
1026  * This routine may be called in atomic context if the RPM_ASYNC flag is set,
1027  * or if pm_runtime_irq_safe() has been called.
1028  */
1029 int __pm_runtime_idle(struct device *dev, int rpmflags)
1030 {
1031 	unsigned long flags;
1032 	int retval;
1033 
1034 	if (rpmflags & RPM_GET_PUT) {
1035 		if (!atomic_dec_and_test(&dev->power.usage_count)) {
1036 			trace_rpm_usage_rcuidle(dev, rpmflags);
1037 			return 0;
1038 		}
1039 	}
1040 
1041 	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
1042 
1043 	spin_lock_irqsave(&dev->power.lock, flags);
1044 	retval = rpm_idle(dev, rpmflags);
1045 	spin_unlock_irqrestore(&dev->power.lock, flags);
1046 
1047 	return retval;
1048 }
1049 EXPORT_SYMBOL_GPL(__pm_runtime_idle);
1050 
1051 /**
1052  * __pm_runtime_suspend - Entry point for runtime put/suspend operations.
1053  * @dev: Device to suspend.
1054  * @rpmflags: Flag bits.
1055  *
1056  * If the RPM_GET_PUT flag is set, decrement the device's usage count and
1057  * return immediately if it is larger than zero.  Then carry out a suspend,
1058  * either synchronous or asynchronous.
1059  *
1060  * This routine may be called in atomic context if the RPM_ASYNC flag is set,
1061  * or if pm_runtime_irq_safe() has been called.
1062  */
1063 int __pm_runtime_suspend(struct device *dev, int rpmflags)
1064 {
1065 	unsigned long flags;
1066 	int retval;
1067 
1068 	if (rpmflags & RPM_GET_PUT) {
1069 		if (!atomic_dec_and_test(&dev->power.usage_count)) {
1070 			trace_rpm_usage_rcuidle(dev, rpmflags);
1071 			return 0;
1072 		}
1073 	}
1074 
1075 	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
1076 
1077 	spin_lock_irqsave(&dev->power.lock, flags);
1078 	retval = rpm_suspend(dev, rpmflags);
1079 	spin_unlock_irqrestore(&dev->power.lock, flags);
1080 
1081 	return retval;
1082 }
1083 EXPORT_SYMBOL_GPL(__pm_runtime_suspend);
1084 
1085 /**
1086  * __pm_runtime_resume - Entry point for runtime resume operations.
1087  * @dev: Device to resume.
1088  * @rpmflags: Flag bits.
1089  *
1090  * If the RPM_GET_PUT flag is set, increment the device's usage count.  Then
1091  * carry out a resume, either synchronous or asynchronous.
1092  *
1093  * This routine may be called in atomic context if the RPM_ASYNC flag is set,
1094  * or if pm_runtime_irq_safe() has been called.
1095  */
1096 int __pm_runtime_resume(struct device *dev, int rpmflags)
1097 {
1098 	unsigned long flags;
1099 	int retval;
1100 
1101 	might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe &&
1102 			dev->power.runtime_status != RPM_ACTIVE);
1103 
1104 	if (rpmflags & RPM_GET_PUT)
1105 		atomic_inc(&dev->power.usage_count);
1106 
1107 	spin_lock_irqsave(&dev->power.lock, flags);
1108 	retval = rpm_resume(dev, rpmflags);
1109 	spin_unlock_irqrestore(&dev->power.lock, flags);
1110 
1111 	return retval;
1112 }
1113 EXPORT_SYMBOL_GPL(__pm_runtime_resume);
1114 
1115 /**
1116  * pm_runtime_get_if_active - Conditionally bump up device usage counter.
1117  * @dev: Device to handle.
1118  * @ign_usage_count: Whether or not to look at the current usage counter value.
1119  *
1120  * Return -EINVAL if runtime PM is disabled for @dev.
1121  *
1122  * Otherwise, if the runtime PM status of @dev is %RPM_ACTIVE and either
1123  * @ign_usage_count is %true or the runtime PM usage counter of @dev is not
1124  * zero, increment the usage counter of @dev and return 1. Otherwise, return 0
1125  * without changing the usage counter.
1126  *
1127  * If @ign_usage_count is %true, this function can be used to prevent suspending
1128  * the device when its runtime PM status is %RPM_ACTIVE.
1129  *
1130  * If @ign_usage_count is %false, this function can be used to prevent
1131  * suspending the device when both its runtime PM status is %RPM_ACTIVE and its
1132  * runtime PM usage counter is not zero.
1133  *
1134  * The caller is responsible for decrementing the runtime PM usage counter of
1135  * @dev after this function has returned a positive value for it.
1136  */
1137 int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count)
1138 {
1139 	unsigned long flags;
1140 	int retval;
1141 
1142 	spin_lock_irqsave(&dev->power.lock, flags);
1143 	if (dev->power.disable_depth > 0) {
1144 		retval = -EINVAL;
1145 	} else if (dev->power.runtime_status != RPM_ACTIVE) {
1146 		retval = 0;
1147 	} else if (ign_usage_count) {
1148 		retval = 1;
1149 		atomic_inc(&dev->power.usage_count);
1150 	} else {
1151 		retval = atomic_inc_not_zero(&dev->power.usage_count);
1152 	}
1153 	trace_rpm_usage_rcuidle(dev, 0);
1154 	spin_unlock_irqrestore(&dev->power.lock, flags);
1155 
1156 	return retval;
1157 }
1158 EXPORT_SYMBOL_GPL(pm_runtime_get_if_active);
1159 
1160 /**
1161  * __pm_runtime_set_status - Set runtime PM status of a device.
1162  * @dev: Device to handle.
1163  * @status: New runtime PM status of the device.
1164  *
1165  * If runtime PM of the device is disabled or its power.runtime_error field is
1166  * different from zero, the status may be changed either to RPM_ACTIVE, or to
1167  * RPM_SUSPENDED, as long as that reflects the actual state of the device.
1168  * However, if the device has a parent and the parent is not active, and the
1169  * parent's power.ignore_children flag is unset, the device's status cannot be
1170  * set to RPM_ACTIVE, so -EBUSY is returned in that case.
1171  *
1172  * If successful, __pm_runtime_set_status() clears the power.runtime_error field
1173  * and the device parent's counter of unsuspended children is modified to
1174  * reflect the new status.  If the new status is RPM_SUSPENDED, an idle
1175  * notification request for the parent is submitted.
1176  *
1177  * If @dev has any suppliers (as reflected by device links to them), and @status
1178  * is RPM_ACTIVE, they will be activated upfront and if the activation of one
1179  * of them fails, the status of @dev will be changed to RPM_SUSPENDED (instead
1180  * of the @status value) and the suppliers will be deacticated on exit.  The
1181  * error returned by the failing supplier activation will be returned in that
1182  * case.
1183  */
1184 int __pm_runtime_set_status(struct device *dev, unsigned int status)
1185 {
1186 	struct device *parent = dev->parent;
1187 	bool notify_parent = false;
1188 	int error = 0;
1189 
1190 	if (status != RPM_ACTIVE && status != RPM_SUSPENDED)
1191 		return -EINVAL;
1192 
1193 	spin_lock_irq(&dev->power.lock);
1194 
1195 	/*
1196 	 * Prevent PM-runtime from being enabled for the device or return an
1197 	 * error if it is enabled already and working.
1198 	 */
1199 	if (dev->power.runtime_error || dev->power.disable_depth)
1200 		dev->power.disable_depth++;
1201 	else
1202 		error = -EAGAIN;
1203 
1204 	spin_unlock_irq(&dev->power.lock);
1205 
1206 	if (error)
1207 		return error;
1208 
1209 	/*
1210 	 * If the new status is RPM_ACTIVE, the suppliers can be activated
1211 	 * upfront regardless of the current status, because next time
1212 	 * rpm_put_suppliers() runs, the rpm_active refcounts of the links
1213 	 * involved will be dropped down to one anyway.
1214 	 */
1215 	if (status == RPM_ACTIVE) {
1216 		int idx = device_links_read_lock();
1217 
1218 		error = rpm_get_suppliers(dev);
1219 		if (error)
1220 			status = RPM_SUSPENDED;
1221 
1222 		device_links_read_unlock(idx);
1223 	}
1224 
1225 	spin_lock_irq(&dev->power.lock);
1226 
1227 	if (dev->power.runtime_status == status || !parent)
1228 		goto out_set;
1229 
1230 	if (status == RPM_SUSPENDED) {
1231 		atomic_add_unless(&parent->power.child_count, -1, 0);
1232 		notify_parent = !parent->power.ignore_children;
1233 	} else {
1234 		spin_lock_nested(&parent->power.lock, SINGLE_DEPTH_NESTING);
1235 
1236 		/*
1237 		 * It is invalid to put an active child under a parent that is
1238 		 * not active, has runtime PM enabled and the
1239 		 * 'power.ignore_children' flag unset.
1240 		 */
1241 		if (!parent->power.disable_depth
1242 		    && !parent->power.ignore_children
1243 		    && parent->power.runtime_status != RPM_ACTIVE) {
1244 			dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n",
1245 				dev_name(dev),
1246 				dev_name(parent));
1247 			error = -EBUSY;
1248 		} else if (dev->power.runtime_status == RPM_SUSPENDED) {
1249 			atomic_inc(&parent->power.child_count);
1250 		}
1251 
1252 		spin_unlock(&parent->power.lock);
1253 
1254 		if (error) {
1255 			status = RPM_SUSPENDED;
1256 			goto out;
1257 		}
1258 	}
1259 
1260  out_set:
1261 	__update_runtime_status(dev, status);
1262 	if (!error)
1263 		dev->power.runtime_error = 0;
1264 
1265  out:
1266 	spin_unlock_irq(&dev->power.lock);
1267 
1268 	if (notify_parent)
1269 		pm_request_idle(parent);
1270 
1271 	if (status == RPM_SUSPENDED) {
1272 		int idx = device_links_read_lock();
1273 
1274 		rpm_put_suppliers(dev);
1275 
1276 		device_links_read_unlock(idx);
1277 	}
1278 
1279 	pm_runtime_enable(dev);
1280 
1281 	return error;
1282 }
1283 EXPORT_SYMBOL_GPL(__pm_runtime_set_status);
1284 
1285 /**
1286  * __pm_runtime_barrier - Cancel pending requests and wait for completions.
1287  * @dev: Device to handle.
1288  *
1289  * Flush all pending requests for the device from pm_wq and wait for all
1290  * runtime PM operations involving the device in progress to complete.
1291  *
1292  * Should be called under dev->power.lock with interrupts disabled.
1293  */
1294 static void __pm_runtime_barrier(struct device *dev)
1295 {
1296 	pm_runtime_deactivate_timer(dev);
1297 
1298 	if (dev->power.request_pending) {
1299 		dev->power.request = RPM_REQ_NONE;
1300 		spin_unlock_irq(&dev->power.lock);
1301 
1302 		cancel_work_sync(&dev->power.work);
1303 
1304 		spin_lock_irq(&dev->power.lock);
1305 		dev->power.request_pending = false;
1306 	}
1307 
1308 	if (dev->power.runtime_status == RPM_SUSPENDING
1309 	    || dev->power.runtime_status == RPM_RESUMING
1310 	    || dev->power.idle_notification) {
1311 		DEFINE_WAIT(wait);
1312 
1313 		/* Suspend, wake-up or idle notification in progress. */
1314 		for (;;) {
1315 			prepare_to_wait(&dev->power.wait_queue, &wait,
1316 					TASK_UNINTERRUPTIBLE);
1317 			if (dev->power.runtime_status != RPM_SUSPENDING
1318 			    && dev->power.runtime_status != RPM_RESUMING
1319 			    && !dev->power.idle_notification)
1320 				break;
1321 			spin_unlock_irq(&dev->power.lock);
1322 
1323 			schedule();
1324 
1325 			spin_lock_irq(&dev->power.lock);
1326 		}
1327 		finish_wait(&dev->power.wait_queue, &wait);
1328 	}
1329 }
1330 
1331 /**
1332  * pm_runtime_barrier - Flush pending requests and wait for completions.
1333  * @dev: Device to handle.
1334  *
1335  * Prevent the device from being suspended by incrementing its usage counter and
1336  * if there's a pending resume request for the device, wake the device up.
1337  * Next, make sure that all pending requests for the device have been flushed
1338  * from pm_wq and wait for all runtime PM operations involving the device in
1339  * progress to complete.
1340  *
1341  * Return value:
1342  * 1, if there was a resume request pending and the device had to be woken up,
1343  * 0, otherwise
1344  */
1345 int pm_runtime_barrier(struct device *dev)
1346 {
1347 	int retval = 0;
1348 
1349 	pm_runtime_get_noresume(dev);
1350 	spin_lock_irq(&dev->power.lock);
1351 
1352 	if (dev->power.request_pending
1353 	    && dev->power.request == RPM_REQ_RESUME) {
1354 		rpm_resume(dev, 0);
1355 		retval = 1;
1356 	}
1357 
1358 	__pm_runtime_barrier(dev);
1359 
1360 	spin_unlock_irq(&dev->power.lock);
1361 	pm_runtime_put_noidle(dev);
1362 
1363 	return retval;
1364 }
1365 EXPORT_SYMBOL_GPL(pm_runtime_barrier);
1366 
1367 /**
1368  * __pm_runtime_disable - Disable runtime PM of a device.
1369  * @dev: Device to handle.
1370  * @check_resume: If set, check if there's a resume request for the device.
1371  *
1372  * Increment power.disable_depth for the device and if it was zero previously,
1373  * cancel all pending runtime PM requests for the device and wait for all
1374  * operations in progress to complete.  The device can be either active or
1375  * suspended after its runtime PM has been disabled.
1376  *
1377  * If @check_resume is set and there's a resume request pending when
1378  * __pm_runtime_disable() is called and power.disable_depth is zero, the
1379  * function will wake up the device before disabling its runtime PM.
1380  */
1381 void __pm_runtime_disable(struct device *dev, bool check_resume)
1382 {
1383 	spin_lock_irq(&dev->power.lock);
1384 
1385 	if (dev->power.disable_depth > 0) {
1386 		dev->power.disable_depth++;
1387 		goto out;
1388 	}
1389 
1390 	/*
1391 	 * Wake up the device if there's a resume request pending, because that
1392 	 * means there probably is some I/O to process and disabling runtime PM
1393 	 * shouldn't prevent the device from processing the I/O.
1394 	 */
1395 	if (check_resume && dev->power.request_pending
1396 	    && dev->power.request == RPM_REQ_RESUME) {
1397 		/*
1398 		 * Prevent suspends and idle notifications from being carried
1399 		 * out after we have woken up the device.
1400 		 */
1401 		pm_runtime_get_noresume(dev);
1402 
1403 		rpm_resume(dev, 0);
1404 
1405 		pm_runtime_put_noidle(dev);
1406 	}
1407 
1408 	/* Update time accounting before disabling PM-runtime. */
1409 	update_pm_runtime_accounting(dev);
1410 
1411 	if (!dev->power.disable_depth++)
1412 		__pm_runtime_barrier(dev);
1413 
1414  out:
1415 	spin_unlock_irq(&dev->power.lock);
1416 }
1417 EXPORT_SYMBOL_GPL(__pm_runtime_disable);
1418 
1419 /**
1420  * pm_runtime_enable - Enable runtime PM of a device.
1421  * @dev: Device to handle.
1422  */
1423 void pm_runtime_enable(struct device *dev)
1424 {
1425 	unsigned long flags;
1426 
1427 	spin_lock_irqsave(&dev->power.lock, flags);
1428 
1429 	if (dev->power.disable_depth > 0) {
1430 		dev->power.disable_depth--;
1431 
1432 		/* About to enable runtime pm, set accounting_timestamp to now */
1433 		if (!dev->power.disable_depth)
1434 			dev->power.accounting_timestamp = ktime_get_mono_fast_ns();
1435 	} else {
1436 		dev_warn(dev, "Unbalanced %s!\n", __func__);
1437 	}
1438 
1439 	WARN(!dev->power.disable_depth &&
1440 	     dev->power.runtime_status == RPM_SUSPENDED &&
1441 	     !dev->power.ignore_children &&
1442 	     atomic_read(&dev->power.child_count) > 0,
1443 	     "Enabling runtime PM for inactive device (%s) with active children\n",
1444 	     dev_name(dev));
1445 
1446 	spin_unlock_irqrestore(&dev->power.lock, flags);
1447 }
1448 EXPORT_SYMBOL_GPL(pm_runtime_enable);
1449 
1450 /**
1451  * pm_runtime_forbid - Block runtime PM of a device.
1452  * @dev: Device to handle.
1453  *
1454  * Increase the device's usage count and clear its power.runtime_auto flag,
1455  * so that it cannot be suspended at run time until pm_runtime_allow() is called
1456  * for it.
1457  */
1458 void pm_runtime_forbid(struct device *dev)
1459 {
1460 	spin_lock_irq(&dev->power.lock);
1461 	if (!dev->power.runtime_auto)
1462 		goto out;
1463 
1464 	dev->power.runtime_auto = false;
1465 	atomic_inc(&dev->power.usage_count);
1466 	rpm_resume(dev, 0);
1467 
1468  out:
1469 	spin_unlock_irq(&dev->power.lock);
1470 }
1471 EXPORT_SYMBOL_GPL(pm_runtime_forbid);
1472 
1473 /**
1474  * pm_runtime_allow - Unblock runtime PM of a device.
1475  * @dev: Device to handle.
1476  *
1477  * Decrease the device's usage count and set its power.runtime_auto flag.
1478  */
1479 void pm_runtime_allow(struct device *dev)
1480 {
1481 	spin_lock_irq(&dev->power.lock);
1482 	if (dev->power.runtime_auto)
1483 		goto out;
1484 
1485 	dev->power.runtime_auto = true;
1486 	if (atomic_dec_and_test(&dev->power.usage_count))
1487 		rpm_idle(dev, RPM_AUTO | RPM_ASYNC);
1488 	else
1489 		trace_rpm_usage_rcuidle(dev, RPM_AUTO | RPM_ASYNC);
1490 
1491  out:
1492 	spin_unlock_irq(&dev->power.lock);
1493 }
1494 EXPORT_SYMBOL_GPL(pm_runtime_allow);
1495 
1496 /**
1497  * pm_runtime_no_callbacks - Ignore runtime PM callbacks for a device.
1498  * @dev: Device to handle.
1499  *
1500  * Set the power.no_callbacks flag, which tells the PM core that this
1501  * device is power-managed through its parent and has no runtime PM
1502  * callbacks of its own.  The runtime sysfs attributes will be removed.
1503  */
1504 void pm_runtime_no_callbacks(struct device *dev)
1505 {
1506 	spin_lock_irq(&dev->power.lock);
1507 	dev->power.no_callbacks = 1;
1508 	spin_unlock_irq(&dev->power.lock);
1509 	if (device_is_registered(dev))
1510 		rpm_sysfs_remove(dev);
1511 }
1512 EXPORT_SYMBOL_GPL(pm_runtime_no_callbacks);
1513 
1514 /**
1515  * pm_runtime_irq_safe - Leave interrupts disabled during callbacks.
1516  * @dev: Device to handle
1517  *
1518  * Set the power.irq_safe flag, which tells the PM core that the
1519  * ->runtime_suspend() and ->runtime_resume() callbacks for this device should
1520  * always be invoked with the spinlock held and interrupts disabled.  It also
1521  * causes the parent's usage counter to be permanently incremented, preventing
1522  * the parent from runtime suspending -- otherwise an irq-safe child might have
1523  * to wait for a non-irq-safe parent.
1524  */
1525 void pm_runtime_irq_safe(struct device *dev)
1526 {
1527 	if (dev->parent)
1528 		pm_runtime_get_sync(dev->parent);
1529 	spin_lock_irq(&dev->power.lock);
1530 	dev->power.irq_safe = 1;
1531 	spin_unlock_irq(&dev->power.lock);
1532 }
1533 EXPORT_SYMBOL_GPL(pm_runtime_irq_safe);
1534 
1535 /**
1536  * update_autosuspend - Handle a change to a device's autosuspend settings.
1537  * @dev: Device to handle.
1538  * @old_delay: The former autosuspend_delay value.
1539  * @old_use: The former use_autosuspend value.
1540  *
1541  * Prevent runtime suspend if the new delay is negative and use_autosuspend is
1542  * set; otherwise allow it.  Send an idle notification if suspends are allowed.
1543  *
1544  * This function must be called under dev->power.lock with interrupts disabled.
1545  */
1546 static void update_autosuspend(struct device *dev, int old_delay, int old_use)
1547 {
1548 	int delay = dev->power.autosuspend_delay;
1549 
1550 	/* Should runtime suspend be prevented now? */
1551 	if (dev->power.use_autosuspend && delay < 0) {
1552 
1553 		/* If it used to be allowed then prevent it. */
1554 		if (!old_use || old_delay >= 0) {
1555 			atomic_inc(&dev->power.usage_count);
1556 			rpm_resume(dev, 0);
1557 		} else {
1558 			trace_rpm_usage_rcuidle(dev, 0);
1559 		}
1560 	}
1561 
1562 	/* Runtime suspend should be allowed now. */
1563 	else {
1564 
1565 		/* If it used to be prevented then allow it. */
1566 		if (old_use && old_delay < 0)
1567 			atomic_dec(&dev->power.usage_count);
1568 
1569 		/* Maybe we can autosuspend now. */
1570 		rpm_idle(dev, RPM_AUTO);
1571 	}
1572 }
1573 
1574 /**
1575  * pm_runtime_set_autosuspend_delay - Set a device's autosuspend_delay value.
1576  * @dev: Device to handle.
1577  * @delay: Value of the new delay in milliseconds.
1578  *
1579  * Set the device's power.autosuspend_delay value.  If it changes to negative
1580  * and the power.use_autosuspend flag is set, prevent runtime suspends.  If it
1581  * changes the other way, allow runtime suspends.
1582  */
1583 void pm_runtime_set_autosuspend_delay(struct device *dev, int delay)
1584 {
1585 	int old_delay, old_use;
1586 
1587 	spin_lock_irq(&dev->power.lock);
1588 	old_delay = dev->power.autosuspend_delay;
1589 	old_use = dev->power.use_autosuspend;
1590 	dev->power.autosuspend_delay = delay;
1591 	update_autosuspend(dev, old_delay, old_use);
1592 	spin_unlock_irq(&dev->power.lock);
1593 }
1594 EXPORT_SYMBOL_GPL(pm_runtime_set_autosuspend_delay);
1595 
1596 /**
1597  * __pm_runtime_use_autosuspend - Set a device's use_autosuspend flag.
1598  * @dev: Device to handle.
1599  * @use: New value for use_autosuspend.
1600  *
1601  * Set the device's power.use_autosuspend flag, and allow or prevent runtime
1602  * suspends as needed.
1603  */
1604 void __pm_runtime_use_autosuspend(struct device *dev, bool use)
1605 {
1606 	int old_delay, old_use;
1607 
1608 	spin_lock_irq(&dev->power.lock);
1609 	old_delay = dev->power.autosuspend_delay;
1610 	old_use = dev->power.use_autosuspend;
1611 	dev->power.use_autosuspend = use;
1612 	update_autosuspend(dev, old_delay, old_use);
1613 	spin_unlock_irq(&dev->power.lock);
1614 }
1615 EXPORT_SYMBOL_GPL(__pm_runtime_use_autosuspend);
1616 
1617 /**
1618  * pm_runtime_init - Initialize runtime PM fields in given device object.
1619  * @dev: Device object to initialize.
1620  */
1621 void pm_runtime_init(struct device *dev)
1622 {
1623 	dev->power.runtime_status = RPM_SUSPENDED;
1624 	dev->power.idle_notification = false;
1625 
1626 	dev->power.disable_depth = 1;
1627 	atomic_set(&dev->power.usage_count, 0);
1628 
1629 	dev->power.runtime_error = 0;
1630 
1631 	atomic_set(&dev->power.child_count, 0);
1632 	pm_suspend_ignore_children(dev, false);
1633 	dev->power.runtime_auto = true;
1634 
1635 	dev->power.request_pending = false;
1636 	dev->power.request = RPM_REQ_NONE;
1637 	dev->power.deferred_resume = false;
1638 	dev->power.needs_force_resume = 0;
1639 	INIT_WORK(&dev->power.work, pm_runtime_work);
1640 
1641 	dev->power.timer_expires = 0;
1642 	hrtimer_init(&dev->power.suspend_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1643 	dev->power.suspend_timer.function = pm_suspend_timer_fn;
1644 
1645 	init_waitqueue_head(&dev->power.wait_queue);
1646 }
1647 
1648 /**
1649  * pm_runtime_reinit - Re-initialize runtime PM fields in given device object.
1650  * @dev: Device object to re-initialize.
1651  */
1652 void pm_runtime_reinit(struct device *dev)
1653 {
1654 	if (!pm_runtime_enabled(dev)) {
1655 		if (dev->power.runtime_status == RPM_ACTIVE)
1656 			pm_runtime_set_suspended(dev);
1657 		if (dev->power.irq_safe) {
1658 			spin_lock_irq(&dev->power.lock);
1659 			dev->power.irq_safe = 0;
1660 			spin_unlock_irq(&dev->power.lock);
1661 			if (dev->parent)
1662 				pm_runtime_put(dev->parent);
1663 		}
1664 	}
1665 }
1666 
1667 /**
1668  * pm_runtime_remove - Prepare for removing a device from device hierarchy.
1669  * @dev: Device object being removed from device hierarchy.
1670  */
1671 void pm_runtime_remove(struct device *dev)
1672 {
1673 	__pm_runtime_disable(dev, false);
1674 	pm_runtime_reinit(dev);
1675 }
1676 
1677 /**
1678  * pm_runtime_get_suppliers - Resume and reference-count supplier devices.
1679  * @dev: Consumer device.
1680  */
1681 void pm_runtime_get_suppliers(struct device *dev)
1682 {
1683 	struct device_link *link;
1684 	int idx;
1685 
1686 	idx = device_links_read_lock();
1687 
1688 	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
1689 				device_links_read_lock_held())
1690 		if (link->flags & DL_FLAG_PM_RUNTIME) {
1691 			link->supplier_preactivated = true;
1692 			pm_runtime_get_sync(link->supplier);
1693 			refcount_inc(&link->rpm_active);
1694 		}
1695 
1696 	device_links_read_unlock(idx);
1697 }
1698 
1699 /**
1700  * pm_runtime_put_suppliers - Drop references to supplier devices.
1701  * @dev: Consumer device.
1702  */
1703 void pm_runtime_put_suppliers(struct device *dev)
1704 {
1705 	struct device_link *link;
1706 	unsigned long flags;
1707 	bool put;
1708 	int idx;
1709 
1710 	idx = device_links_read_lock();
1711 
1712 	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
1713 				device_links_read_lock_held())
1714 		if (link->supplier_preactivated) {
1715 			link->supplier_preactivated = false;
1716 			spin_lock_irqsave(&dev->power.lock, flags);
1717 			put = pm_runtime_status_suspended(dev) &&
1718 			      refcount_dec_not_one(&link->rpm_active);
1719 			spin_unlock_irqrestore(&dev->power.lock, flags);
1720 			if (put)
1721 				pm_runtime_put(link->supplier);
1722 		}
1723 
1724 	device_links_read_unlock(idx);
1725 }
1726 
1727 void pm_runtime_new_link(struct device *dev)
1728 {
1729 	spin_lock_irq(&dev->power.lock);
1730 	dev->power.links_count++;
1731 	spin_unlock_irq(&dev->power.lock);
1732 }
1733 
1734 static void pm_runtime_drop_link_count(struct device *dev)
1735 {
1736 	spin_lock_irq(&dev->power.lock);
1737 	WARN_ON(dev->power.links_count == 0);
1738 	dev->power.links_count--;
1739 	spin_unlock_irq(&dev->power.lock);
1740 }
1741 
1742 /**
1743  * pm_runtime_drop_link - Prepare for device link removal.
1744  * @link: Device link going away.
1745  *
1746  * Drop the link count of the consumer end of @link and decrement the supplier
1747  * device's runtime PM usage counter as many times as needed to drop all of the
1748  * PM runtime reference to it from the consumer.
1749  */
1750 void pm_runtime_drop_link(struct device_link *link)
1751 {
1752 	if (!(link->flags & DL_FLAG_PM_RUNTIME))
1753 		return;
1754 
1755 	pm_runtime_drop_link_count(link->consumer);
1756 
1757 	while (refcount_dec_not_one(&link->rpm_active))
1758 		pm_runtime_put(link->supplier);
1759 }
1760 
1761 static bool pm_runtime_need_not_resume(struct device *dev)
1762 {
1763 	return atomic_read(&dev->power.usage_count) <= 1 &&
1764 		(atomic_read(&dev->power.child_count) == 0 ||
1765 		 dev->power.ignore_children);
1766 }
1767 
1768 /**
1769  * pm_runtime_force_suspend - Force a device into suspend state if needed.
1770  * @dev: Device to suspend.
1771  *
1772  * Disable runtime PM so we safely can check the device's runtime PM status and
1773  * if it is active, invoke its ->runtime_suspend callback to suspend it and
1774  * change its runtime PM status field to RPM_SUSPENDED.  Also, if the device's
1775  * usage and children counters don't indicate that the device was in use before
1776  * the system-wide transition under way, decrement its parent's children counter
1777  * (if there is a parent).  Keep runtime PM disabled to preserve the state
1778  * unless we encounter errors.
1779  *
1780  * Typically this function may be invoked from a system suspend callback to make
1781  * sure the device is put into low power state and it should only be used during
1782  * system-wide PM transitions to sleep states.  It assumes that the analogous
1783  * pm_runtime_force_resume() will be used to resume the device.
1784  */
1785 int pm_runtime_force_suspend(struct device *dev)
1786 {
1787 	int (*callback)(struct device *);
1788 	int ret;
1789 
1790 	pm_runtime_disable(dev);
1791 	if (pm_runtime_status_suspended(dev))
1792 		return 0;
1793 
1794 	callback = RPM_GET_CALLBACK(dev, runtime_suspend);
1795 
1796 	ret = callback ? callback(dev) : 0;
1797 	if (ret)
1798 		goto err;
1799 
1800 	/*
1801 	 * If the device can stay in suspend after the system-wide transition
1802 	 * to the working state that will follow, drop the children counter of
1803 	 * its parent, but set its status to RPM_SUSPENDED anyway in case this
1804 	 * function will be called again for it in the meantime.
1805 	 */
1806 	if (pm_runtime_need_not_resume(dev)) {
1807 		pm_runtime_set_suspended(dev);
1808 	} else {
1809 		__update_runtime_status(dev, RPM_SUSPENDED);
1810 		dev->power.needs_force_resume = 1;
1811 	}
1812 
1813 	return 0;
1814 
1815 err:
1816 	pm_runtime_enable(dev);
1817 	return ret;
1818 }
1819 EXPORT_SYMBOL_GPL(pm_runtime_force_suspend);
1820 
1821 /**
1822  * pm_runtime_force_resume - Force a device into resume state if needed.
1823  * @dev: Device to resume.
1824  *
1825  * Prior invoking this function we expect the user to have brought the device
1826  * into low power state by a call to pm_runtime_force_suspend(). Here we reverse
1827  * those actions and bring the device into full power, if it is expected to be
1828  * used on system resume.  In the other case, we defer the resume to be managed
1829  * via runtime PM.
1830  *
1831  * Typically this function may be invoked from a system resume callback.
1832  */
1833 int pm_runtime_force_resume(struct device *dev)
1834 {
1835 	int (*callback)(struct device *);
1836 	int ret = 0;
1837 
1838 	if (!pm_runtime_status_suspended(dev) || !dev->power.needs_force_resume)
1839 		goto out;
1840 
1841 	/*
1842 	 * The value of the parent's children counter is correct already, so
1843 	 * just update the status of the device.
1844 	 */
1845 	__update_runtime_status(dev, RPM_ACTIVE);
1846 
1847 	callback = RPM_GET_CALLBACK(dev, runtime_resume);
1848 
1849 	ret = callback ? callback(dev) : 0;
1850 	if (ret) {
1851 		pm_runtime_set_suspended(dev);
1852 		goto out;
1853 	}
1854 
1855 	pm_runtime_mark_last_busy(dev);
1856 out:
1857 	dev->power.needs_force_resume = 0;
1858 	pm_runtime_enable(dev);
1859 	return ret;
1860 }
1861 EXPORT_SYMBOL_GPL(pm_runtime_force_resume);
1862