1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * drivers/base/power/runtime.c - Helper functions for device runtime PM 4 * 5 * Copyright (c) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc. 6 * Copyright (C) 2010 Alan Stern <stern@rowland.harvard.edu> 7 */ 8 #include <linux/sched/mm.h> 9 #include <linux/ktime.h> 10 #include <linux/hrtimer.h> 11 #include <linux/export.h> 12 #include <linux/pm_runtime.h> 13 #include <linux/pm_wakeirq.h> 14 #include <trace/events/rpm.h> 15 16 #include "../base.h" 17 #include "power.h" 18 19 typedef int (*pm_callback_t)(struct device *); 20 21 static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset) 22 { 23 pm_callback_t cb; 24 const struct dev_pm_ops *ops; 25 26 if (dev->pm_domain) 27 ops = &dev->pm_domain->ops; 28 else if (dev->type && dev->type->pm) 29 ops = dev->type->pm; 30 else if (dev->class && dev->class->pm) 31 ops = dev->class->pm; 32 else if (dev->bus && dev->bus->pm) 33 ops = dev->bus->pm; 34 else 35 ops = NULL; 36 37 if (ops) 38 cb = *(pm_callback_t *)((void *)ops + cb_offset); 39 else 40 cb = NULL; 41 42 if (!cb && dev->driver && dev->driver->pm) 43 cb = *(pm_callback_t *)((void *)dev->driver->pm + cb_offset); 44 45 return cb; 46 } 47 48 #define RPM_GET_CALLBACK(dev, callback) \ 49 __rpm_get_callback(dev, offsetof(struct dev_pm_ops, callback)) 50 51 static int rpm_resume(struct device *dev, int rpmflags); 52 static int rpm_suspend(struct device *dev, int rpmflags); 53 54 /** 55 * update_pm_runtime_accounting - Update the time accounting of power states 56 * @dev: Device to update the accounting for 57 * 58 * In order to be able to have time accounting of the various power states 59 * (as used by programs such as PowerTOP to show the effectiveness of runtime 60 * PM), we need to track the time spent in each state. 61 * update_pm_runtime_accounting must be called each time before the 62 * runtime_status field is updated, to account the time in the old state 63 * correctly. 64 */ 65 static void update_pm_runtime_accounting(struct device *dev) 66 { 67 u64 now, last, delta; 68 69 if (dev->power.disable_depth > 0) 70 return; 71 72 last = dev->power.accounting_timestamp; 73 74 now = ktime_get_mono_fast_ns(); 75 dev->power.accounting_timestamp = now; 76 77 /* 78 * Because ktime_get_mono_fast_ns() is not monotonic during 79 * timekeeping updates, ensure that 'now' is after the last saved 80 * timesptamp. 81 */ 82 if (now < last) 83 return; 84 85 delta = now - last; 86 87 if (dev->power.runtime_status == RPM_SUSPENDED) 88 dev->power.suspended_time += delta; 89 else 90 dev->power.active_time += delta; 91 } 92 93 static void __update_runtime_status(struct device *dev, enum rpm_status status) 94 { 95 update_pm_runtime_accounting(dev); 96 dev->power.runtime_status = status; 97 } 98 99 static u64 rpm_get_accounted_time(struct device *dev, bool suspended) 100 { 101 u64 time; 102 unsigned long flags; 103 104 spin_lock_irqsave(&dev->power.lock, flags); 105 106 update_pm_runtime_accounting(dev); 107 time = suspended ? dev->power.suspended_time : dev->power.active_time; 108 109 spin_unlock_irqrestore(&dev->power.lock, flags); 110 111 return time; 112 } 113 114 u64 pm_runtime_active_time(struct device *dev) 115 { 116 return rpm_get_accounted_time(dev, false); 117 } 118 119 u64 pm_runtime_suspended_time(struct device *dev) 120 { 121 return rpm_get_accounted_time(dev, true); 122 } 123 EXPORT_SYMBOL_GPL(pm_runtime_suspended_time); 124 125 /** 126 * pm_runtime_deactivate_timer - Deactivate given device's suspend timer. 127 * @dev: Device to handle. 128 */ 129 static void pm_runtime_deactivate_timer(struct device *dev) 130 { 131 if (dev->power.timer_expires > 0) { 132 hrtimer_try_to_cancel(&dev->power.suspend_timer); 133 dev->power.timer_expires = 0; 134 } 135 } 136 137 /** 138 * pm_runtime_cancel_pending - Deactivate suspend timer and cancel requests. 139 * @dev: Device to handle. 140 */ 141 static void pm_runtime_cancel_pending(struct device *dev) 142 { 143 pm_runtime_deactivate_timer(dev); 144 /* 145 * In case there's a request pending, make sure its work function will 146 * return without doing anything. 147 */ 148 dev->power.request = RPM_REQ_NONE; 149 } 150 151 /* 152 * pm_runtime_autosuspend_expiration - Get a device's autosuspend-delay expiration time. 153 * @dev: Device to handle. 154 * 155 * Compute the autosuspend-delay expiration time based on the device's 156 * power.last_busy time. If the delay has already expired or is disabled 157 * (negative) or the power.use_autosuspend flag isn't set, return 0. 158 * Otherwise return the expiration time in nanoseconds (adjusted to be nonzero). 159 * 160 * This function may be called either with or without dev->power.lock held. 161 * Either way it can be racy, since power.last_busy may be updated at any time. 162 */ 163 u64 pm_runtime_autosuspend_expiration(struct device *dev) 164 { 165 int autosuspend_delay; 166 u64 expires; 167 168 if (!dev->power.use_autosuspend) 169 return 0; 170 171 autosuspend_delay = READ_ONCE(dev->power.autosuspend_delay); 172 if (autosuspend_delay < 0) 173 return 0; 174 175 expires = READ_ONCE(dev->power.last_busy); 176 expires += (u64)autosuspend_delay * NSEC_PER_MSEC; 177 if (expires > ktime_get_mono_fast_ns()) 178 return expires; /* Expires in the future */ 179 180 return 0; 181 } 182 EXPORT_SYMBOL_GPL(pm_runtime_autosuspend_expiration); 183 184 static int dev_memalloc_noio(struct device *dev, void *data) 185 { 186 return dev->power.memalloc_noio; 187 } 188 189 /* 190 * pm_runtime_set_memalloc_noio - Set a device's memalloc_noio flag. 191 * @dev: Device to handle. 192 * @enable: True for setting the flag and False for clearing the flag. 193 * 194 * Set the flag for all devices in the path from the device to the 195 * root device in the device tree if @enable is true, otherwise clear 196 * the flag for devices in the path whose siblings don't set the flag. 197 * 198 * The function should only be called by block device, or network 199 * device driver for solving the deadlock problem during runtime 200 * resume/suspend: 201 * 202 * If memory allocation with GFP_KERNEL is called inside runtime 203 * resume/suspend callback of any one of its ancestors(or the 204 * block device itself), the deadlock may be triggered inside the 205 * memory allocation since it might not complete until the block 206 * device becomes active and the involed page I/O finishes. The 207 * situation is pointed out first by Alan Stern. Network device 208 * are involved in iSCSI kind of situation. 209 * 210 * The lock of dev_hotplug_mutex is held in the function for handling 211 * hotplug race because pm_runtime_set_memalloc_noio() may be called 212 * in async probe(). 213 * 214 * The function should be called between device_add() and device_del() 215 * on the affected device(block/network device). 216 */ 217 void pm_runtime_set_memalloc_noio(struct device *dev, bool enable) 218 { 219 static DEFINE_MUTEX(dev_hotplug_mutex); 220 221 mutex_lock(&dev_hotplug_mutex); 222 for (;;) { 223 bool enabled; 224 225 /* hold power lock since bitfield is not SMP-safe. */ 226 spin_lock_irq(&dev->power.lock); 227 enabled = dev->power.memalloc_noio; 228 dev->power.memalloc_noio = enable; 229 spin_unlock_irq(&dev->power.lock); 230 231 /* 232 * not need to enable ancestors any more if the device 233 * has been enabled. 234 */ 235 if (enabled && enable) 236 break; 237 238 dev = dev->parent; 239 240 /* 241 * clear flag of the parent device only if all the 242 * children don't set the flag because ancestor's 243 * flag was set by any one of the descendants. 244 */ 245 if (!dev || (!enable && 246 device_for_each_child(dev, NULL, 247 dev_memalloc_noio))) 248 break; 249 } 250 mutex_unlock(&dev_hotplug_mutex); 251 } 252 EXPORT_SYMBOL_GPL(pm_runtime_set_memalloc_noio); 253 254 /** 255 * rpm_check_suspend_allowed - Test whether a device may be suspended. 256 * @dev: Device to test. 257 */ 258 static int rpm_check_suspend_allowed(struct device *dev) 259 { 260 int retval = 0; 261 262 if (dev->power.runtime_error) 263 retval = -EINVAL; 264 else if (dev->power.disable_depth > 0) 265 retval = -EACCES; 266 else if (atomic_read(&dev->power.usage_count) > 0) 267 retval = -EAGAIN; 268 else if (!dev->power.ignore_children && 269 atomic_read(&dev->power.child_count)) 270 retval = -EBUSY; 271 272 /* Pending resume requests take precedence over suspends. */ 273 else if ((dev->power.deferred_resume 274 && dev->power.runtime_status == RPM_SUSPENDING) 275 || (dev->power.request_pending 276 && dev->power.request == RPM_REQ_RESUME)) 277 retval = -EAGAIN; 278 else if (__dev_pm_qos_resume_latency(dev) == 0) 279 retval = -EPERM; 280 else if (dev->power.runtime_status == RPM_SUSPENDED) 281 retval = 1; 282 283 return retval; 284 } 285 286 static int rpm_get_suppliers(struct device *dev) 287 { 288 struct device_link *link; 289 290 list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, 291 device_links_read_lock_held()) { 292 int retval; 293 294 if (!(link->flags & DL_FLAG_PM_RUNTIME)) 295 continue; 296 297 retval = pm_runtime_get_sync(link->supplier); 298 /* Ignore suppliers with disabled runtime PM. */ 299 if (retval < 0 && retval != -EACCES) { 300 pm_runtime_put_noidle(link->supplier); 301 return retval; 302 } 303 refcount_inc(&link->rpm_active); 304 } 305 return 0; 306 } 307 308 /** 309 * pm_runtime_release_supplier - Drop references to device link's supplier. 310 * @link: Target device link. 311 * 312 * Drop all runtime PM references associated with @link to its supplier device. 313 */ 314 void pm_runtime_release_supplier(struct device_link *link) 315 { 316 struct device *supplier = link->supplier; 317 318 /* 319 * The additional power.usage_count check is a safety net in case 320 * the rpm_active refcount becomes saturated, in which case 321 * refcount_dec_not_one() would return true forever, but it is not 322 * strictly necessary. 323 */ 324 while (refcount_dec_not_one(&link->rpm_active) && 325 atomic_read(&supplier->power.usage_count) > 0) 326 pm_runtime_put_noidle(supplier); 327 } 328 329 static void __rpm_put_suppliers(struct device *dev, bool try_to_suspend) 330 { 331 struct device_link *link; 332 333 list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, 334 device_links_read_lock_held()) { 335 pm_runtime_release_supplier(link); 336 if (try_to_suspend) 337 pm_request_idle(link->supplier); 338 } 339 } 340 341 static void rpm_put_suppliers(struct device *dev) 342 { 343 __rpm_put_suppliers(dev, true); 344 } 345 346 static void rpm_suspend_suppliers(struct device *dev) 347 { 348 struct device_link *link; 349 int idx = device_links_read_lock(); 350 351 list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, 352 device_links_read_lock_held()) 353 pm_request_idle(link->supplier); 354 355 device_links_read_unlock(idx); 356 } 357 358 /** 359 * __rpm_callback - Run a given runtime PM callback for a given device. 360 * @cb: Runtime PM callback to run. 361 * @dev: Device to run the callback for. 362 */ 363 static int __rpm_callback(int (*cb)(struct device *), struct device *dev) 364 __releases(&dev->power.lock) __acquires(&dev->power.lock) 365 { 366 int retval = 0, idx; 367 bool use_links = dev->power.links_count > 0; 368 369 if (dev->power.irq_safe) { 370 spin_unlock(&dev->power.lock); 371 } else { 372 spin_unlock_irq(&dev->power.lock); 373 374 /* 375 * Resume suppliers if necessary. 376 * 377 * The device's runtime PM status cannot change until this 378 * routine returns, so it is safe to read the status outside of 379 * the lock. 380 */ 381 if (use_links && dev->power.runtime_status == RPM_RESUMING) { 382 idx = device_links_read_lock(); 383 384 retval = rpm_get_suppliers(dev); 385 if (retval) { 386 rpm_put_suppliers(dev); 387 goto fail; 388 } 389 390 device_links_read_unlock(idx); 391 } 392 } 393 394 if (cb) 395 retval = cb(dev); 396 397 if (dev->power.irq_safe) { 398 spin_lock(&dev->power.lock); 399 } else { 400 /* 401 * If the device is suspending and the callback has returned 402 * success, drop the usage counters of the suppliers that have 403 * been reference counted on its resume. 404 * 405 * Do that if resume fails too. 406 */ 407 if (use_links 408 && ((dev->power.runtime_status == RPM_SUSPENDING && !retval) 409 || (dev->power.runtime_status == RPM_RESUMING && retval))) { 410 idx = device_links_read_lock(); 411 412 __rpm_put_suppliers(dev, false); 413 414 fail: 415 device_links_read_unlock(idx); 416 } 417 418 spin_lock_irq(&dev->power.lock); 419 } 420 421 return retval; 422 } 423 424 /** 425 * rpm_idle - Notify device bus type if the device can be suspended. 426 * @dev: Device to notify the bus type about. 427 * @rpmflags: Flag bits. 428 * 429 * Check if the device's runtime PM status allows it to be suspended. If 430 * another idle notification has been started earlier, return immediately. If 431 * the RPM_ASYNC flag is set then queue an idle-notification request; otherwise 432 * run the ->runtime_idle() callback directly. If the ->runtime_idle callback 433 * doesn't exist or if it returns 0, call rpm_suspend with the RPM_AUTO flag. 434 * 435 * This function must be called under dev->power.lock with interrupts disabled. 436 */ 437 static int rpm_idle(struct device *dev, int rpmflags) 438 { 439 int (*callback)(struct device *); 440 int retval; 441 442 trace_rpm_idle_rcuidle(dev, rpmflags); 443 retval = rpm_check_suspend_allowed(dev); 444 if (retval < 0) 445 ; /* Conditions are wrong. */ 446 447 /* Idle notifications are allowed only in the RPM_ACTIVE state. */ 448 else if (dev->power.runtime_status != RPM_ACTIVE) 449 retval = -EAGAIN; 450 451 /* 452 * Any pending request other than an idle notification takes 453 * precedence over us, except that the timer may be running. 454 */ 455 else if (dev->power.request_pending && 456 dev->power.request > RPM_REQ_IDLE) 457 retval = -EAGAIN; 458 459 /* Act as though RPM_NOWAIT is always set. */ 460 else if (dev->power.idle_notification) 461 retval = -EINPROGRESS; 462 if (retval) 463 goto out; 464 465 /* Pending requests need to be canceled. */ 466 dev->power.request = RPM_REQ_NONE; 467 468 callback = RPM_GET_CALLBACK(dev, runtime_idle); 469 470 /* If no callback assume success. */ 471 if (!callback || dev->power.no_callbacks) 472 goto out; 473 474 /* Carry out an asynchronous or a synchronous idle notification. */ 475 if (rpmflags & RPM_ASYNC) { 476 dev->power.request = RPM_REQ_IDLE; 477 if (!dev->power.request_pending) { 478 dev->power.request_pending = true; 479 queue_work(pm_wq, &dev->power.work); 480 } 481 trace_rpm_return_int_rcuidle(dev, _THIS_IP_, 0); 482 return 0; 483 } 484 485 dev->power.idle_notification = true; 486 487 retval = __rpm_callback(callback, dev); 488 489 dev->power.idle_notification = false; 490 wake_up_all(&dev->power.wait_queue); 491 492 out: 493 trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval); 494 return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO); 495 } 496 497 /** 498 * rpm_callback - Run a given runtime PM callback for a given device. 499 * @cb: Runtime PM callback to run. 500 * @dev: Device to run the callback for. 501 */ 502 static int rpm_callback(int (*cb)(struct device *), struct device *dev) 503 { 504 int retval; 505 506 if (dev->power.memalloc_noio) { 507 unsigned int noio_flag; 508 509 /* 510 * Deadlock might be caused if memory allocation with 511 * GFP_KERNEL happens inside runtime_suspend and 512 * runtime_resume callbacks of one block device's 513 * ancestor or the block device itself. Network 514 * device might be thought as part of iSCSI block 515 * device, so network device and its ancestor should 516 * be marked as memalloc_noio too. 517 */ 518 noio_flag = memalloc_noio_save(); 519 retval = __rpm_callback(cb, dev); 520 memalloc_noio_restore(noio_flag); 521 } else { 522 retval = __rpm_callback(cb, dev); 523 } 524 525 dev->power.runtime_error = retval; 526 return retval != -EACCES ? retval : -EIO; 527 } 528 529 /** 530 * rpm_suspend - Carry out runtime suspend of given device. 531 * @dev: Device to suspend. 532 * @rpmflags: Flag bits. 533 * 534 * Check if the device's runtime PM status allows it to be suspended. 535 * Cancel a pending idle notification, autosuspend or suspend. If 536 * another suspend has been started earlier, either return immediately 537 * or wait for it to finish, depending on the RPM_NOWAIT and RPM_ASYNC 538 * flags. If the RPM_ASYNC flag is set then queue a suspend request; 539 * otherwise run the ->runtime_suspend() callback directly. When 540 * ->runtime_suspend succeeded, if a deferred resume was requested while 541 * the callback was running then carry it out, otherwise send an idle 542 * notification for its parent (if the suspend succeeded and both 543 * ignore_children of parent->power and irq_safe of dev->power are not set). 544 * If ->runtime_suspend failed with -EAGAIN or -EBUSY, and if the RPM_AUTO 545 * flag is set and the next autosuspend-delay expiration time is in the 546 * future, schedule another autosuspend attempt. 547 * 548 * This function must be called under dev->power.lock with interrupts disabled. 549 */ 550 static int rpm_suspend(struct device *dev, int rpmflags) 551 __releases(&dev->power.lock) __acquires(&dev->power.lock) 552 { 553 int (*callback)(struct device *); 554 struct device *parent = NULL; 555 int retval; 556 557 trace_rpm_suspend_rcuidle(dev, rpmflags); 558 559 repeat: 560 retval = rpm_check_suspend_allowed(dev); 561 if (retval < 0) 562 goto out; /* Conditions are wrong. */ 563 564 /* Synchronous suspends are not allowed in the RPM_RESUMING state. */ 565 if (dev->power.runtime_status == RPM_RESUMING && !(rpmflags & RPM_ASYNC)) 566 retval = -EAGAIN; 567 if (retval) 568 goto out; 569 570 /* If the autosuspend_delay time hasn't expired yet, reschedule. */ 571 if ((rpmflags & RPM_AUTO) 572 && dev->power.runtime_status != RPM_SUSPENDING) { 573 u64 expires = pm_runtime_autosuspend_expiration(dev); 574 575 if (expires != 0) { 576 /* Pending requests need to be canceled. */ 577 dev->power.request = RPM_REQ_NONE; 578 579 /* 580 * Optimization: If the timer is already running and is 581 * set to expire at or before the autosuspend delay, 582 * avoid the overhead of resetting it. Just let it 583 * expire; pm_suspend_timer_fn() will take care of the 584 * rest. 585 */ 586 if (!(dev->power.timer_expires && 587 dev->power.timer_expires <= expires)) { 588 /* 589 * We add a slack of 25% to gather wakeups 590 * without sacrificing the granularity. 591 */ 592 u64 slack = (u64)READ_ONCE(dev->power.autosuspend_delay) * 593 (NSEC_PER_MSEC >> 2); 594 595 dev->power.timer_expires = expires; 596 hrtimer_start_range_ns(&dev->power.suspend_timer, 597 ns_to_ktime(expires), 598 slack, 599 HRTIMER_MODE_ABS); 600 } 601 dev->power.timer_autosuspends = 1; 602 goto out; 603 } 604 } 605 606 /* Other scheduled or pending requests need to be canceled. */ 607 pm_runtime_cancel_pending(dev); 608 609 if (dev->power.runtime_status == RPM_SUSPENDING) { 610 DEFINE_WAIT(wait); 611 612 if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) { 613 retval = -EINPROGRESS; 614 goto out; 615 } 616 617 if (dev->power.irq_safe) { 618 spin_unlock(&dev->power.lock); 619 620 cpu_relax(); 621 622 spin_lock(&dev->power.lock); 623 goto repeat; 624 } 625 626 /* Wait for the other suspend running in parallel with us. */ 627 for (;;) { 628 prepare_to_wait(&dev->power.wait_queue, &wait, 629 TASK_UNINTERRUPTIBLE); 630 if (dev->power.runtime_status != RPM_SUSPENDING) 631 break; 632 633 spin_unlock_irq(&dev->power.lock); 634 635 schedule(); 636 637 spin_lock_irq(&dev->power.lock); 638 } 639 finish_wait(&dev->power.wait_queue, &wait); 640 goto repeat; 641 } 642 643 if (dev->power.no_callbacks) 644 goto no_callback; /* Assume success. */ 645 646 /* Carry out an asynchronous or a synchronous suspend. */ 647 if (rpmflags & RPM_ASYNC) { 648 dev->power.request = (rpmflags & RPM_AUTO) ? 649 RPM_REQ_AUTOSUSPEND : RPM_REQ_SUSPEND; 650 if (!dev->power.request_pending) { 651 dev->power.request_pending = true; 652 queue_work(pm_wq, &dev->power.work); 653 } 654 goto out; 655 } 656 657 __update_runtime_status(dev, RPM_SUSPENDING); 658 659 callback = RPM_GET_CALLBACK(dev, runtime_suspend); 660 661 dev_pm_enable_wake_irq_check(dev, true); 662 retval = rpm_callback(callback, dev); 663 if (retval) 664 goto fail; 665 666 no_callback: 667 __update_runtime_status(dev, RPM_SUSPENDED); 668 pm_runtime_deactivate_timer(dev); 669 670 if (dev->parent) { 671 parent = dev->parent; 672 atomic_add_unless(&parent->power.child_count, -1, 0); 673 } 674 wake_up_all(&dev->power.wait_queue); 675 676 if (dev->power.deferred_resume) { 677 dev->power.deferred_resume = false; 678 rpm_resume(dev, 0); 679 retval = -EAGAIN; 680 goto out; 681 } 682 683 if (dev->power.irq_safe) 684 goto out; 685 686 /* Maybe the parent is now able to suspend. */ 687 if (parent && !parent->power.ignore_children) { 688 spin_unlock(&dev->power.lock); 689 690 spin_lock(&parent->power.lock); 691 rpm_idle(parent, RPM_ASYNC); 692 spin_unlock(&parent->power.lock); 693 694 spin_lock(&dev->power.lock); 695 } 696 /* Maybe the suppliers are now able to suspend. */ 697 if (dev->power.links_count > 0) { 698 spin_unlock_irq(&dev->power.lock); 699 700 rpm_suspend_suppliers(dev); 701 702 spin_lock_irq(&dev->power.lock); 703 } 704 705 out: 706 trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval); 707 708 return retval; 709 710 fail: 711 dev_pm_disable_wake_irq_check(dev); 712 __update_runtime_status(dev, RPM_ACTIVE); 713 dev->power.deferred_resume = false; 714 wake_up_all(&dev->power.wait_queue); 715 716 if (retval == -EAGAIN || retval == -EBUSY) { 717 dev->power.runtime_error = 0; 718 719 /* 720 * If the callback routine failed an autosuspend, and 721 * if the last_busy time has been updated so that there 722 * is a new autosuspend expiration time, automatically 723 * reschedule another autosuspend. 724 */ 725 if ((rpmflags & RPM_AUTO) && 726 pm_runtime_autosuspend_expiration(dev) != 0) 727 goto repeat; 728 } else { 729 pm_runtime_cancel_pending(dev); 730 } 731 goto out; 732 } 733 734 /** 735 * rpm_resume - Carry out runtime resume of given device. 736 * @dev: Device to resume. 737 * @rpmflags: Flag bits. 738 * 739 * Check if the device's runtime PM status allows it to be resumed. Cancel 740 * any scheduled or pending requests. If another resume has been started 741 * earlier, either return immediately or wait for it to finish, depending on the 742 * RPM_NOWAIT and RPM_ASYNC flags. Similarly, if there's a suspend running in 743 * parallel with this function, either tell the other process to resume after 744 * suspending (deferred_resume) or wait for it to finish. If the RPM_ASYNC 745 * flag is set then queue a resume request; otherwise run the 746 * ->runtime_resume() callback directly. Queue an idle notification for the 747 * device if the resume succeeded. 748 * 749 * This function must be called under dev->power.lock with interrupts disabled. 750 */ 751 static int rpm_resume(struct device *dev, int rpmflags) 752 __releases(&dev->power.lock) __acquires(&dev->power.lock) 753 { 754 int (*callback)(struct device *); 755 struct device *parent = NULL; 756 int retval = 0; 757 758 trace_rpm_resume_rcuidle(dev, rpmflags); 759 760 repeat: 761 if (dev->power.runtime_error) 762 retval = -EINVAL; 763 else if (dev->power.disable_depth == 1 && dev->power.is_suspended 764 && dev->power.runtime_status == RPM_ACTIVE) 765 retval = 1; 766 else if (dev->power.disable_depth > 0) 767 retval = -EACCES; 768 if (retval) 769 goto out; 770 771 /* 772 * Other scheduled or pending requests need to be canceled. Small 773 * optimization: If an autosuspend timer is running, leave it running 774 * rather than cancelling it now only to restart it again in the near 775 * future. 776 */ 777 dev->power.request = RPM_REQ_NONE; 778 if (!dev->power.timer_autosuspends) 779 pm_runtime_deactivate_timer(dev); 780 781 if (dev->power.runtime_status == RPM_ACTIVE) { 782 retval = 1; 783 goto out; 784 } 785 786 if (dev->power.runtime_status == RPM_RESUMING 787 || dev->power.runtime_status == RPM_SUSPENDING) { 788 DEFINE_WAIT(wait); 789 790 if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) { 791 if (dev->power.runtime_status == RPM_SUSPENDING) 792 dev->power.deferred_resume = true; 793 else 794 retval = -EINPROGRESS; 795 goto out; 796 } 797 798 if (dev->power.irq_safe) { 799 spin_unlock(&dev->power.lock); 800 801 cpu_relax(); 802 803 spin_lock(&dev->power.lock); 804 goto repeat; 805 } 806 807 /* Wait for the operation carried out in parallel with us. */ 808 for (;;) { 809 prepare_to_wait(&dev->power.wait_queue, &wait, 810 TASK_UNINTERRUPTIBLE); 811 if (dev->power.runtime_status != RPM_RESUMING 812 && dev->power.runtime_status != RPM_SUSPENDING) 813 break; 814 815 spin_unlock_irq(&dev->power.lock); 816 817 schedule(); 818 819 spin_lock_irq(&dev->power.lock); 820 } 821 finish_wait(&dev->power.wait_queue, &wait); 822 goto repeat; 823 } 824 825 /* 826 * See if we can skip waking up the parent. This is safe only if 827 * power.no_callbacks is set, because otherwise we don't know whether 828 * the resume will actually succeed. 829 */ 830 if (dev->power.no_callbacks && !parent && dev->parent) { 831 spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING); 832 if (dev->parent->power.disable_depth > 0 833 || dev->parent->power.ignore_children 834 || dev->parent->power.runtime_status == RPM_ACTIVE) { 835 atomic_inc(&dev->parent->power.child_count); 836 spin_unlock(&dev->parent->power.lock); 837 retval = 1; 838 goto no_callback; /* Assume success. */ 839 } 840 spin_unlock(&dev->parent->power.lock); 841 } 842 843 /* Carry out an asynchronous or a synchronous resume. */ 844 if (rpmflags & RPM_ASYNC) { 845 dev->power.request = RPM_REQ_RESUME; 846 if (!dev->power.request_pending) { 847 dev->power.request_pending = true; 848 queue_work(pm_wq, &dev->power.work); 849 } 850 retval = 0; 851 goto out; 852 } 853 854 if (!parent && dev->parent) { 855 /* 856 * Increment the parent's usage counter and resume it if 857 * necessary. Not needed if dev is irq-safe; then the 858 * parent is permanently resumed. 859 */ 860 parent = dev->parent; 861 if (dev->power.irq_safe) 862 goto skip_parent; 863 spin_unlock(&dev->power.lock); 864 865 pm_runtime_get_noresume(parent); 866 867 spin_lock(&parent->power.lock); 868 /* 869 * Resume the parent if it has runtime PM enabled and not been 870 * set to ignore its children. 871 */ 872 if (!parent->power.disable_depth 873 && !parent->power.ignore_children) { 874 rpm_resume(parent, 0); 875 if (parent->power.runtime_status != RPM_ACTIVE) 876 retval = -EBUSY; 877 } 878 spin_unlock(&parent->power.lock); 879 880 spin_lock(&dev->power.lock); 881 if (retval) 882 goto out; 883 goto repeat; 884 } 885 skip_parent: 886 887 if (dev->power.no_callbacks) 888 goto no_callback; /* Assume success. */ 889 890 __update_runtime_status(dev, RPM_RESUMING); 891 892 callback = RPM_GET_CALLBACK(dev, runtime_resume); 893 894 dev_pm_disable_wake_irq_check(dev); 895 retval = rpm_callback(callback, dev); 896 if (retval) { 897 __update_runtime_status(dev, RPM_SUSPENDED); 898 pm_runtime_cancel_pending(dev); 899 dev_pm_enable_wake_irq_check(dev, false); 900 } else { 901 no_callback: 902 __update_runtime_status(dev, RPM_ACTIVE); 903 pm_runtime_mark_last_busy(dev); 904 if (parent) 905 atomic_inc(&parent->power.child_count); 906 } 907 wake_up_all(&dev->power.wait_queue); 908 909 if (retval >= 0) 910 rpm_idle(dev, RPM_ASYNC); 911 912 out: 913 if (parent && !dev->power.irq_safe) { 914 spin_unlock_irq(&dev->power.lock); 915 916 pm_runtime_put(parent); 917 918 spin_lock_irq(&dev->power.lock); 919 } 920 921 trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval); 922 923 return retval; 924 } 925 926 /** 927 * pm_runtime_work - Universal runtime PM work function. 928 * @work: Work structure used for scheduling the execution of this function. 929 * 930 * Use @work to get the device object the work is to be done for, determine what 931 * is to be done and execute the appropriate runtime PM function. 932 */ 933 static void pm_runtime_work(struct work_struct *work) 934 { 935 struct device *dev = container_of(work, struct device, power.work); 936 enum rpm_request req; 937 938 spin_lock_irq(&dev->power.lock); 939 940 if (!dev->power.request_pending) 941 goto out; 942 943 req = dev->power.request; 944 dev->power.request = RPM_REQ_NONE; 945 dev->power.request_pending = false; 946 947 switch (req) { 948 case RPM_REQ_NONE: 949 break; 950 case RPM_REQ_IDLE: 951 rpm_idle(dev, RPM_NOWAIT); 952 break; 953 case RPM_REQ_SUSPEND: 954 rpm_suspend(dev, RPM_NOWAIT); 955 break; 956 case RPM_REQ_AUTOSUSPEND: 957 rpm_suspend(dev, RPM_NOWAIT | RPM_AUTO); 958 break; 959 case RPM_REQ_RESUME: 960 rpm_resume(dev, RPM_NOWAIT); 961 break; 962 } 963 964 out: 965 spin_unlock_irq(&dev->power.lock); 966 } 967 968 /** 969 * pm_suspend_timer_fn - Timer function for pm_schedule_suspend(). 970 * @timer: hrtimer used by pm_schedule_suspend(). 971 * 972 * Check if the time is right and queue a suspend request. 973 */ 974 static enum hrtimer_restart pm_suspend_timer_fn(struct hrtimer *timer) 975 { 976 struct device *dev = container_of(timer, struct device, power.suspend_timer); 977 unsigned long flags; 978 u64 expires; 979 980 spin_lock_irqsave(&dev->power.lock, flags); 981 982 expires = dev->power.timer_expires; 983 /* 984 * If 'expires' is after the current time, we've been called 985 * too early. 986 */ 987 if (expires > 0 && expires < ktime_get_mono_fast_ns()) { 988 dev->power.timer_expires = 0; 989 rpm_suspend(dev, dev->power.timer_autosuspends ? 990 (RPM_ASYNC | RPM_AUTO) : RPM_ASYNC); 991 } 992 993 spin_unlock_irqrestore(&dev->power.lock, flags); 994 995 return HRTIMER_NORESTART; 996 } 997 998 /** 999 * pm_schedule_suspend - Set up a timer to submit a suspend request in future. 1000 * @dev: Device to suspend. 1001 * @delay: Time to wait before submitting a suspend request, in milliseconds. 1002 */ 1003 int pm_schedule_suspend(struct device *dev, unsigned int delay) 1004 { 1005 unsigned long flags; 1006 u64 expires; 1007 int retval; 1008 1009 spin_lock_irqsave(&dev->power.lock, flags); 1010 1011 if (!delay) { 1012 retval = rpm_suspend(dev, RPM_ASYNC); 1013 goto out; 1014 } 1015 1016 retval = rpm_check_suspend_allowed(dev); 1017 if (retval) 1018 goto out; 1019 1020 /* Other scheduled or pending requests need to be canceled. */ 1021 pm_runtime_cancel_pending(dev); 1022 1023 expires = ktime_get_mono_fast_ns() + (u64)delay * NSEC_PER_MSEC; 1024 dev->power.timer_expires = expires; 1025 dev->power.timer_autosuspends = 0; 1026 hrtimer_start(&dev->power.suspend_timer, expires, HRTIMER_MODE_ABS); 1027 1028 out: 1029 spin_unlock_irqrestore(&dev->power.lock, flags); 1030 1031 return retval; 1032 } 1033 EXPORT_SYMBOL_GPL(pm_schedule_suspend); 1034 1035 /** 1036 * __pm_runtime_idle - Entry point for runtime idle operations. 1037 * @dev: Device to send idle notification for. 1038 * @rpmflags: Flag bits. 1039 * 1040 * If the RPM_GET_PUT flag is set, decrement the device's usage count and 1041 * return immediately if it is larger than zero. Then carry out an idle 1042 * notification, either synchronous or asynchronous. 1043 * 1044 * This routine may be called in atomic context if the RPM_ASYNC flag is set, 1045 * or if pm_runtime_irq_safe() has been called. 1046 */ 1047 int __pm_runtime_idle(struct device *dev, int rpmflags) 1048 { 1049 unsigned long flags; 1050 int retval; 1051 1052 if (rpmflags & RPM_GET_PUT) { 1053 if (!atomic_dec_and_test(&dev->power.usage_count)) { 1054 trace_rpm_usage_rcuidle(dev, rpmflags); 1055 return 0; 1056 } 1057 } 1058 1059 might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe); 1060 1061 spin_lock_irqsave(&dev->power.lock, flags); 1062 retval = rpm_idle(dev, rpmflags); 1063 spin_unlock_irqrestore(&dev->power.lock, flags); 1064 1065 return retval; 1066 } 1067 EXPORT_SYMBOL_GPL(__pm_runtime_idle); 1068 1069 /** 1070 * __pm_runtime_suspend - Entry point for runtime put/suspend operations. 1071 * @dev: Device to suspend. 1072 * @rpmflags: Flag bits. 1073 * 1074 * If the RPM_GET_PUT flag is set, decrement the device's usage count and 1075 * return immediately if it is larger than zero. Then carry out a suspend, 1076 * either synchronous or asynchronous. 1077 * 1078 * This routine may be called in atomic context if the RPM_ASYNC flag is set, 1079 * or if pm_runtime_irq_safe() has been called. 1080 */ 1081 int __pm_runtime_suspend(struct device *dev, int rpmflags) 1082 { 1083 unsigned long flags; 1084 int retval; 1085 1086 if (rpmflags & RPM_GET_PUT) { 1087 if (!atomic_dec_and_test(&dev->power.usage_count)) { 1088 trace_rpm_usage_rcuidle(dev, rpmflags); 1089 return 0; 1090 } 1091 } 1092 1093 might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe); 1094 1095 spin_lock_irqsave(&dev->power.lock, flags); 1096 retval = rpm_suspend(dev, rpmflags); 1097 spin_unlock_irqrestore(&dev->power.lock, flags); 1098 1099 return retval; 1100 } 1101 EXPORT_SYMBOL_GPL(__pm_runtime_suspend); 1102 1103 /** 1104 * __pm_runtime_resume - Entry point for runtime resume operations. 1105 * @dev: Device to resume. 1106 * @rpmflags: Flag bits. 1107 * 1108 * If the RPM_GET_PUT flag is set, increment the device's usage count. Then 1109 * carry out a resume, either synchronous or asynchronous. 1110 * 1111 * This routine may be called in atomic context if the RPM_ASYNC flag is set, 1112 * or if pm_runtime_irq_safe() has been called. 1113 */ 1114 int __pm_runtime_resume(struct device *dev, int rpmflags) 1115 { 1116 unsigned long flags; 1117 int retval; 1118 1119 might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe && 1120 dev->power.runtime_status != RPM_ACTIVE); 1121 1122 if (rpmflags & RPM_GET_PUT) 1123 atomic_inc(&dev->power.usage_count); 1124 1125 spin_lock_irqsave(&dev->power.lock, flags); 1126 retval = rpm_resume(dev, rpmflags); 1127 spin_unlock_irqrestore(&dev->power.lock, flags); 1128 1129 return retval; 1130 } 1131 EXPORT_SYMBOL_GPL(__pm_runtime_resume); 1132 1133 /** 1134 * pm_runtime_get_if_active - Conditionally bump up device usage counter. 1135 * @dev: Device to handle. 1136 * @ign_usage_count: Whether or not to look at the current usage counter value. 1137 * 1138 * Return -EINVAL if runtime PM is disabled for @dev. 1139 * 1140 * Otherwise, if the runtime PM status of @dev is %RPM_ACTIVE and either 1141 * @ign_usage_count is %true or the runtime PM usage counter of @dev is not 1142 * zero, increment the usage counter of @dev and return 1. Otherwise, return 0 1143 * without changing the usage counter. 1144 * 1145 * If @ign_usage_count is %true, this function can be used to prevent suspending 1146 * the device when its runtime PM status is %RPM_ACTIVE. 1147 * 1148 * If @ign_usage_count is %false, this function can be used to prevent 1149 * suspending the device when both its runtime PM status is %RPM_ACTIVE and its 1150 * runtime PM usage counter is not zero. 1151 * 1152 * The caller is responsible for decrementing the runtime PM usage counter of 1153 * @dev after this function has returned a positive value for it. 1154 */ 1155 int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count) 1156 { 1157 unsigned long flags; 1158 int retval; 1159 1160 spin_lock_irqsave(&dev->power.lock, flags); 1161 if (dev->power.disable_depth > 0) { 1162 retval = -EINVAL; 1163 } else if (dev->power.runtime_status != RPM_ACTIVE) { 1164 retval = 0; 1165 } else if (ign_usage_count) { 1166 retval = 1; 1167 atomic_inc(&dev->power.usage_count); 1168 } else { 1169 retval = atomic_inc_not_zero(&dev->power.usage_count); 1170 } 1171 trace_rpm_usage_rcuidle(dev, 0); 1172 spin_unlock_irqrestore(&dev->power.lock, flags); 1173 1174 return retval; 1175 } 1176 EXPORT_SYMBOL_GPL(pm_runtime_get_if_active); 1177 1178 /** 1179 * __pm_runtime_set_status - Set runtime PM status of a device. 1180 * @dev: Device to handle. 1181 * @status: New runtime PM status of the device. 1182 * 1183 * If runtime PM of the device is disabled or its power.runtime_error field is 1184 * different from zero, the status may be changed either to RPM_ACTIVE, or to 1185 * RPM_SUSPENDED, as long as that reflects the actual state of the device. 1186 * However, if the device has a parent and the parent is not active, and the 1187 * parent's power.ignore_children flag is unset, the device's status cannot be 1188 * set to RPM_ACTIVE, so -EBUSY is returned in that case. 1189 * 1190 * If successful, __pm_runtime_set_status() clears the power.runtime_error field 1191 * and the device parent's counter of unsuspended children is modified to 1192 * reflect the new status. If the new status is RPM_SUSPENDED, an idle 1193 * notification request for the parent is submitted. 1194 * 1195 * If @dev has any suppliers (as reflected by device links to them), and @status 1196 * is RPM_ACTIVE, they will be activated upfront and if the activation of one 1197 * of them fails, the status of @dev will be changed to RPM_SUSPENDED (instead 1198 * of the @status value) and the suppliers will be deacticated on exit. The 1199 * error returned by the failing supplier activation will be returned in that 1200 * case. 1201 */ 1202 int __pm_runtime_set_status(struct device *dev, unsigned int status) 1203 { 1204 struct device *parent = dev->parent; 1205 bool notify_parent = false; 1206 int error = 0; 1207 1208 if (status != RPM_ACTIVE && status != RPM_SUSPENDED) 1209 return -EINVAL; 1210 1211 spin_lock_irq(&dev->power.lock); 1212 1213 /* 1214 * Prevent PM-runtime from being enabled for the device or return an 1215 * error if it is enabled already and working. 1216 */ 1217 if (dev->power.runtime_error || dev->power.disable_depth) 1218 dev->power.disable_depth++; 1219 else 1220 error = -EAGAIN; 1221 1222 spin_unlock_irq(&dev->power.lock); 1223 1224 if (error) 1225 return error; 1226 1227 /* 1228 * If the new status is RPM_ACTIVE, the suppliers can be activated 1229 * upfront regardless of the current status, because next time 1230 * rpm_put_suppliers() runs, the rpm_active refcounts of the links 1231 * involved will be dropped down to one anyway. 1232 */ 1233 if (status == RPM_ACTIVE) { 1234 int idx = device_links_read_lock(); 1235 1236 error = rpm_get_suppliers(dev); 1237 if (error) 1238 status = RPM_SUSPENDED; 1239 1240 device_links_read_unlock(idx); 1241 } 1242 1243 spin_lock_irq(&dev->power.lock); 1244 1245 if (dev->power.runtime_status == status || !parent) 1246 goto out_set; 1247 1248 if (status == RPM_SUSPENDED) { 1249 atomic_add_unless(&parent->power.child_count, -1, 0); 1250 notify_parent = !parent->power.ignore_children; 1251 } else { 1252 spin_lock_nested(&parent->power.lock, SINGLE_DEPTH_NESTING); 1253 1254 /* 1255 * It is invalid to put an active child under a parent that is 1256 * not active, has runtime PM enabled and the 1257 * 'power.ignore_children' flag unset. 1258 */ 1259 if (!parent->power.disable_depth 1260 && !parent->power.ignore_children 1261 && parent->power.runtime_status != RPM_ACTIVE) { 1262 dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n", 1263 dev_name(dev), 1264 dev_name(parent)); 1265 error = -EBUSY; 1266 } else if (dev->power.runtime_status == RPM_SUSPENDED) { 1267 atomic_inc(&parent->power.child_count); 1268 } 1269 1270 spin_unlock(&parent->power.lock); 1271 1272 if (error) { 1273 status = RPM_SUSPENDED; 1274 goto out; 1275 } 1276 } 1277 1278 out_set: 1279 __update_runtime_status(dev, status); 1280 if (!error) 1281 dev->power.runtime_error = 0; 1282 1283 out: 1284 spin_unlock_irq(&dev->power.lock); 1285 1286 if (notify_parent) 1287 pm_request_idle(parent); 1288 1289 if (status == RPM_SUSPENDED) { 1290 int idx = device_links_read_lock(); 1291 1292 rpm_put_suppliers(dev); 1293 1294 device_links_read_unlock(idx); 1295 } 1296 1297 pm_runtime_enable(dev); 1298 1299 return error; 1300 } 1301 EXPORT_SYMBOL_GPL(__pm_runtime_set_status); 1302 1303 /** 1304 * __pm_runtime_barrier - Cancel pending requests and wait for completions. 1305 * @dev: Device to handle. 1306 * 1307 * Flush all pending requests for the device from pm_wq and wait for all 1308 * runtime PM operations involving the device in progress to complete. 1309 * 1310 * Should be called under dev->power.lock with interrupts disabled. 1311 */ 1312 static void __pm_runtime_barrier(struct device *dev) 1313 { 1314 pm_runtime_deactivate_timer(dev); 1315 1316 if (dev->power.request_pending) { 1317 dev->power.request = RPM_REQ_NONE; 1318 spin_unlock_irq(&dev->power.lock); 1319 1320 cancel_work_sync(&dev->power.work); 1321 1322 spin_lock_irq(&dev->power.lock); 1323 dev->power.request_pending = false; 1324 } 1325 1326 if (dev->power.runtime_status == RPM_SUSPENDING 1327 || dev->power.runtime_status == RPM_RESUMING 1328 || dev->power.idle_notification) { 1329 DEFINE_WAIT(wait); 1330 1331 /* Suspend, wake-up or idle notification in progress. */ 1332 for (;;) { 1333 prepare_to_wait(&dev->power.wait_queue, &wait, 1334 TASK_UNINTERRUPTIBLE); 1335 if (dev->power.runtime_status != RPM_SUSPENDING 1336 && dev->power.runtime_status != RPM_RESUMING 1337 && !dev->power.idle_notification) 1338 break; 1339 spin_unlock_irq(&dev->power.lock); 1340 1341 schedule(); 1342 1343 spin_lock_irq(&dev->power.lock); 1344 } 1345 finish_wait(&dev->power.wait_queue, &wait); 1346 } 1347 } 1348 1349 /** 1350 * pm_runtime_barrier - Flush pending requests and wait for completions. 1351 * @dev: Device to handle. 1352 * 1353 * Prevent the device from being suspended by incrementing its usage counter and 1354 * if there's a pending resume request for the device, wake the device up. 1355 * Next, make sure that all pending requests for the device have been flushed 1356 * from pm_wq and wait for all runtime PM operations involving the device in 1357 * progress to complete. 1358 * 1359 * Return value: 1360 * 1, if there was a resume request pending and the device had to be woken up, 1361 * 0, otherwise 1362 */ 1363 int pm_runtime_barrier(struct device *dev) 1364 { 1365 int retval = 0; 1366 1367 pm_runtime_get_noresume(dev); 1368 spin_lock_irq(&dev->power.lock); 1369 1370 if (dev->power.request_pending 1371 && dev->power.request == RPM_REQ_RESUME) { 1372 rpm_resume(dev, 0); 1373 retval = 1; 1374 } 1375 1376 __pm_runtime_barrier(dev); 1377 1378 spin_unlock_irq(&dev->power.lock); 1379 pm_runtime_put_noidle(dev); 1380 1381 return retval; 1382 } 1383 EXPORT_SYMBOL_GPL(pm_runtime_barrier); 1384 1385 /** 1386 * __pm_runtime_disable - Disable runtime PM of a device. 1387 * @dev: Device to handle. 1388 * @check_resume: If set, check if there's a resume request for the device. 1389 * 1390 * Increment power.disable_depth for the device and if it was zero previously, 1391 * cancel all pending runtime PM requests for the device and wait for all 1392 * operations in progress to complete. The device can be either active or 1393 * suspended after its runtime PM has been disabled. 1394 * 1395 * If @check_resume is set and there's a resume request pending when 1396 * __pm_runtime_disable() is called and power.disable_depth is zero, the 1397 * function will wake up the device before disabling its runtime PM. 1398 */ 1399 void __pm_runtime_disable(struct device *dev, bool check_resume) 1400 { 1401 spin_lock_irq(&dev->power.lock); 1402 1403 if (dev->power.disable_depth > 0) { 1404 dev->power.disable_depth++; 1405 goto out; 1406 } 1407 1408 /* 1409 * Wake up the device if there's a resume request pending, because that 1410 * means there probably is some I/O to process and disabling runtime PM 1411 * shouldn't prevent the device from processing the I/O. 1412 */ 1413 if (check_resume && dev->power.request_pending 1414 && dev->power.request == RPM_REQ_RESUME) { 1415 /* 1416 * Prevent suspends and idle notifications from being carried 1417 * out after we have woken up the device. 1418 */ 1419 pm_runtime_get_noresume(dev); 1420 1421 rpm_resume(dev, 0); 1422 1423 pm_runtime_put_noidle(dev); 1424 } 1425 1426 /* Update time accounting before disabling PM-runtime. */ 1427 update_pm_runtime_accounting(dev); 1428 1429 if (!dev->power.disable_depth++) 1430 __pm_runtime_barrier(dev); 1431 1432 out: 1433 spin_unlock_irq(&dev->power.lock); 1434 } 1435 EXPORT_SYMBOL_GPL(__pm_runtime_disable); 1436 1437 /** 1438 * pm_runtime_enable - Enable runtime PM of a device. 1439 * @dev: Device to handle. 1440 */ 1441 void pm_runtime_enable(struct device *dev) 1442 { 1443 unsigned long flags; 1444 1445 spin_lock_irqsave(&dev->power.lock, flags); 1446 1447 if (dev->power.disable_depth > 0) { 1448 dev->power.disable_depth--; 1449 1450 /* About to enable runtime pm, set accounting_timestamp to now */ 1451 if (!dev->power.disable_depth) 1452 dev->power.accounting_timestamp = ktime_get_mono_fast_ns(); 1453 } else { 1454 dev_warn(dev, "Unbalanced %s!\n", __func__); 1455 } 1456 1457 WARN(!dev->power.disable_depth && 1458 dev->power.runtime_status == RPM_SUSPENDED && 1459 !dev->power.ignore_children && 1460 atomic_read(&dev->power.child_count) > 0, 1461 "Enabling runtime PM for inactive device (%s) with active children\n", 1462 dev_name(dev)); 1463 1464 spin_unlock_irqrestore(&dev->power.lock, flags); 1465 } 1466 EXPORT_SYMBOL_GPL(pm_runtime_enable); 1467 1468 static void pm_runtime_disable_action(void *data) 1469 { 1470 pm_runtime_disable(data); 1471 } 1472 1473 /** 1474 * devm_pm_runtime_enable - devres-enabled version of pm_runtime_enable. 1475 * @dev: Device to handle. 1476 */ 1477 int devm_pm_runtime_enable(struct device *dev) 1478 { 1479 pm_runtime_enable(dev); 1480 1481 return devm_add_action_or_reset(dev, pm_runtime_disable_action, dev); 1482 } 1483 EXPORT_SYMBOL_GPL(devm_pm_runtime_enable); 1484 1485 /** 1486 * pm_runtime_forbid - Block runtime PM of a device. 1487 * @dev: Device to handle. 1488 * 1489 * Increase the device's usage count and clear its power.runtime_auto flag, 1490 * so that it cannot be suspended at run time until pm_runtime_allow() is called 1491 * for it. 1492 */ 1493 void pm_runtime_forbid(struct device *dev) 1494 { 1495 spin_lock_irq(&dev->power.lock); 1496 if (!dev->power.runtime_auto) 1497 goto out; 1498 1499 dev->power.runtime_auto = false; 1500 atomic_inc(&dev->power.usage_count); 1501 rpm_resume(dev, 0); 1502 1503 out: 1504 spin_unlock_irq(&dev->power.lock); 1505 } 1506 EXPORT_SYMBOL_GPL(pm_runtime_forbid); 1507 1508 /** 1509 * pm_runtime_allow - Unblock runtime PM of a device. 1510 * @dev: Device to handle. 1511 * 1512 * Decrease the device's usage count and set its power.runtime_auto flag. 1513 */ 1514 void pm_runtime_allow(struct device *dev) 1515 { 1516 spin_lock_irq(&dev->power.lock); 1517 if (dev->power.runtime_auto) 1518 goto out; 1519 1520 dev->power.runtime_auto = true; 1521 if (atomic_dec_and_test(&dev->power.usage_count)) 1522 rpm_idle(dev, RPM_AUTO | RPM_ASYNC); 1523 else 1524 trace_rpm_usage_rcuidle(dev, RPM_AUTO | RPM_ASYNC); 1525 1526 out: 1527 spin_unlock_irq(&dev->power.lock); 1528 } 1529 EXPORT_SYMBOL_GPL(pm_runtime_allow); 1530 1531 /** 1532 * pm_runtime_no_callbacks - Ignore runtime PM callbacks for a device. 1533 * @dev: Device to handle. 1534 * 1535 * Set the power.no_callbacks flag, which tells the PM core that this 1536 * device is power-managed through its parent and has no runtime PM 1537 * callbacks of its own. The runtime sysfs attributes will be removed. 1538 */ 1539 void pm_runtime_no_callbacks(struct device *dev) 1540 { 1541 spin_lock_irq(&dev->power.lock); 1542 dev->power.no_callbacks = 1; 1543 spin_unlock_irq(&dev->power.lock); 1544 if (device_is_registered(dev)) 1545 rpm_sysfs_remove(dev); 1546 } 1547 EXPORT_SYMBOL_GPL(pm_runtime_no_callbacks); 1548 1549 /** 1550 * pm_runtime_irq_safe - Leave interrupts disabled during callbacks. 1551 * @dev: Device to handle 1552 * 1553 * Set the power.irq_safe flag, which tells the PM core that the 1554 * ->runtime_suspend() and ->runtime_resume() callbacks for this device should 1555 * always be invoked with the spinlock held and interrupts disabled. It also 1556 * causes the parent's usage counter to be permanently incremented, preventing 1557 * the parent from runtime suspending -- otherwise an irq-safe child might have 1558 * to wait for a non-irq-safe parent. 1559 */ 1560 void pm_runtime_irq_safe(struct device *dev) 1561 { 1562 if (dev->parent) 1563 pm_runtime_get_sync(dev->parent); 1564 spin_lock_irq(&dev->power.lock); 1565 dev->power.irq_safe = 1; 1566 spin_unlock_irq(&dev->power.lock); 1567 } 1568 EXPORT_SYMBOL_GPL(pm_runtime_irq_safe); 1569 1570 /** 1571 * update_autosuspend - Handle a change to a device's autosuspend settings. 1572 * @dev: Device to handle. 1573 * @old_delay: The former autosuspend_delay value. 1574 * @old_use: The former use_autosuspend value. 1575 * 1576 * Prevent runtime suspend if the new delay is negative and use_autosuspend is 1577 * set; otherwise allow it. Send an idle notification if suspends are allowed. 1578 * 1579 * This function must be called under dev->power.lock with interrupts disabled. 1580 */ 1581 static void update_autosuspend(struct device *dev, int old_delay, int old_use) 1582 { 1583 int delay = dev->power.autosuspend_delay; 1584 1585 /* Should runtime suspend be prevented now? */ 1586 if (dev->power.use_autosuspend && delay < 0) { 1587 1588 /* If it used to be allowed then prevent it. */ 1589 if (!old_use || old_delay >= 0) { 1590 atomic_inc(&dev->power.usage_count); 1591 rpm_resume(dev, 0); 1592 } else { 1593 trace_rpm_usage_rcuidle(dev, 0); 1594 } 1595 } 1596 1597 /* Runtime suspend should be allowed now. */ 1598 else { 1599 1600 /* If it used to be prevented then allow it. */ 1601 if (old_use && old_delay < 0) 1602 atomic_dec(&dev->power.usage_count); 1603 1604 /* Maybe we can autosuspend now. */ 1605 rpm_idle(dev, RPM_AUTO); 1606 } 1607 } 1608 1609 /** 1610 * pm_runtime_set_autosuspend_delay - Set a device's autosuspend_delay value. 1611 * @dev: Device to handle. 1612 * @delay: Value of the new delay in milliseconds. 1613 * 1614 * Set the device's power.autosuspend_delay value. If it changes to negative 1615 * and the power.use_autosuspend flag is set, prevent runtime suspends. If it 1616 * changes the other way, allow runtime suspends. 1617 */ 1618 void pm_runtime_set_autosuspend_delay(struct device *dev, int delay) 1619 { 1620 int old_delay, old_use; 1621 1622 spin_lock_irq(&dev->power.lock); 1623 old_delay = dev->power.autosuspend_delay; 1624 old_use = dev->power.use_autosuspend; 1625 dev->power.autosuspend_delay = delay; 1626 update_autosuspend(dev, old_delay, old_use); 1627 spin_unlock_irq(&dev->power.lock); 1628 } 1629 EXPORT_SYMBOL_GPL(pm_runtime_set_autosuspend_delay); 1630 1631 /** 1632 * __pm_runtime_use_autosuspend - Set a device's use_autosuspend flag. 1633 * @dev: Device to handle. 1634 * @use: New value for use_autosuspend. 1635 * 1636 * Set the device's power.use_autosuspend flag, and allow or prevent runtime 1637 * suspends as needed. 1638 */ 1639 void __pm_runtime_use_autosuspend(struct device *dev, bool use) 1640 { 1641 int old_delay, old_use; 1642 1643 spin_lock_irq(&dev->power.lock); 1644 old_delay = dev->power.autosuspend_delay; 1645 old_use = dev->power.use_autosuspend; 1646 dev->power.use_autosuspend = use; 1647 update_autosuspend(dev, old_delay, old_use); 1648 spin_unlock_irq(&dev->power.lock); 1649 } 1650 EXPORT_SYMBOL_GPL(__pm_runtime_use_autosuspend); 1651 1652 /** 1653 * pm_runtime_init - Initialize runtime PM fields in given device object. 1654 * @dev: Device object to initialize. 1655 */ 1656 void pm_runtime_init(struct device *dev) 1657 { 1658 dev->power.runtime_status = RPM_SUSPENDED; 1659 dev->power.idle_notification = false; 1660 1661 dev->power.disable_depth = 1; 1662 atomic_set(&dev->power.usage_count, 0); 1663 1664 dev->power.runtime_error = 0; 1665 1666 atomic_set(&dev->power.child_count, 0); 1667 pm_suspend_ignore_children(dev, false); 1668 dev->power.runtime_auto = true; 1669 1670 dev->power.request_pending = false; 1671 dev->power.request = RPM_REQ_NONE; 1672 dev->power.deferred_resume = false; 1673 dev->power.needs_force_resume = 0; 1674 INIT_WORK(&dev->power.work, pm_runtime_work); 1675 1676 dev->power.timer_expires = 0; 1677 hrtimer_init(&dev->power.suspend_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 1678 dev->power.suspend_timer.function = pm_suspend_timer_fn; 1679 1680 init_waitqueue_head(&dev->power.wait_queue); 1681 } 1682 1683 /** 1684 * pm_runtime_reinit - Re-initialize runtime PM fields in given device object. 1685 * @dev: Device object to re-initialize. 1686 */ 1687 void pm_runtime_reinit(struct device *dev) 1688 { 1689 if (!pm_runtime_enabled(dev)) { 1690 if (dev->power.runtime_status == RPM_ACTIVE) 1691 pm_runtime_set_suspended(dev); 1692 if (dev->power.irq_safe) { 1693 spin_lock_irq(&dev->power.lock); 1694 dev->power.irq_safe = 0; 1695 spin_unlock_irq(&dev->power.lock); 1696 if (dev->parent) 1697 pm_runtime_put(dev->parent); 1698 } 1699 } 1700 } 1701 1702 /** 1703 * pm_runtime_remove - Prepare for removing a device from device hierarchy. 1704 * @dev: Device object being removed from device hierarchy. 1705 */ 1706 void pm_runtime_remove(struct device *dev) 1707 { 1708 __pm_runtime_disable(dev, false); 1709 pm_runtime_reinit(dev); 1710 } 1711 1712 /** 1713 * pm_runtime_get_suppliers - Resume and reference-count supplier devices. 1714 * @dev: Consumer device. 1715 */ 1716 void pm_runtime_get_suppliers(struct device *dev) 1717 { 1718 struct device_link *link; 1719 int idx; 1720 1721 idx = device_links_read_lock(); 1722 1723 list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, 1724 device_links_read_lock_held()) 1725 if (link->flags & DL_FLAG_PM_RUNTIME) { 1726 link->supplier_preactivated = true; 1727 pm_runtime_get_sync(link->supplier); 1728 refcount_inc(&link->rpm_active); 1729 } 1730 1731 device_links_read_unlock(idx); 1732 } 1733 1734 /** 1735 * pm_runtime_put_suppliers - Drop references to supplier devices. 1736 * @dev: Consumer device. 1737 */ 1738 void pm_runtime_put_suppliers(struct device *dev) 1739 { 1740 struct device_link *link; 1741 unsigned long flags; 1742 bool put; 1743 int idx; 1744 1745 idx = device_links_read_lock(); 1746 1747 list_for_each_entry_rcu(link, &dev->links.suppliers, c_node, 1748 device_links_read_lock_held()) 1749 if (link->supplier_preactivated) { 1750 link->supplier_preactivated = false; 1751 spin_lock_irqsave(&dev->power.lock, flags); 1752 put = pm_runtime_status_suspended(dev) && 1753 refcount_dec_not_one(&link->rpm_active); 1754 spin_unlock_irqrestore(&dev->power.lock, flags); 1755 if (put) 1756 pm_runtime_put(link->supplier); 1757 } 1758 1759 device_links_read_unlock(idx); 1760 } 1761 1762 void pm_runtime_new_link(struct device *dev) 1763 { 1764 spin_lock_irq(&dev->power.lock); 1765 dev->power.links_count++; 1766 spin_unlock_irq(&dev->power.lock); 1767 } 1768 1769 static void pm_runtime_drop_link_count(struct device *dev) 1770 { 1771 spin_lock_irq(&dev->power.lock); 1772 WARN_ON(dev->power.links_count == 0); 1773 dev->power.links_count--; 1774 spin_unlock_irq(&dev->power.lock); 1775 } 1776 1777 /** 1778 * pm_runtime_drop_link - Prepare for device link removal. 1779 * @link: Device link going away. 1780 * 1781 * Drop the link count of the consumer end of @link and decrement the supplier 1782 * device's runtime PM usage counter as many times as needed to drop all of the 1783 * PM runtime reference to it from the consumer. 1784 */ 1785 void pm_runtime_drop_link(struct device_link *link) 1786 { 1787 if (!(link->flags & DL_FLAG_PM_RUNTIME)) 1788 return; 1789 1790 pm_runtime_drop_link_count(link->consumer); 1791 pm_runtime_release_supplier(link); 1792 pm_request_idle(link->supplier); 1793 } 1794 1795 static bool pm_runtime_need_not_resume(struct device *dev) 1796 { 1797 return atomic_read(&dev->power.usage_count) <= 1 && 1798 (atomic_read(&dev->power.child_count) == 0 || 1799 dev->power.ignore_children); 1800 } 1801 1802 /** 1803 * pm_runtime_force_suspend - Force a device into suspend state if needed. 1804 * @dev: Device to suspend. 1805 * 1806 * Disable runtime PM so we safely can check the device's runtime PM status and 1807 * if it is active, invoke its ->runtime_suspend callback to suspend it and 1808 * change its runtime PM status field to RPM_SUSPENDED. Also, if the device's 1809 * usage and children counters don't indicate that the device was in use before 1810 * the system-wide transition under way, decrement its parent's children counter 1811 * (if there is a parent). Keep runtime PM disabled to preserve the state 1812 * unless we encounter errors. 1813 * 1814 * Typically this function may be invoked from a system suspend callback to make 1815 * sure the device is put into low power state and it should only be used during 1816 * system-wide PM transitions to sleep states. It assumes that the analogous 1817 * pm_runtime_force_resume() will be used to resume the device. 1818 */ 1819 int pm_runtime_force_suspend(struct device *dev) 1820 { 1821 int (*callback)(struct device *); 1822 int ret; 1823 1824 pm_runtime_disable(dev); 1825 if (pm_runtime_status_suspended(dev)) 1826 return 0; 1827 1828 callback = RPM_GET_CALLBACK(dev, runtime_suspend); 1829 1830 ret = callback ? callback(dev) : 0; 1831 if (ret) 1832 goto err; 1833 1834 /* 1835 * If the device can stay in suspend after the system-wide transition 1836 * to the working state that will follow, drop the children counter of 1837 * its parent, but set its status to RPM_SUSPENDED anyway in case this 1838 * function will be called again for it in the meantime. 1839 */ 1840 if (pm_runtime_need_not_resume(dev)) { 1841 pm_runtime_set_suspended(dev); 1842 } else { 1843 __update_runtime_status(dev, RPM_SUSPENDED); 1844 dev->power.needs_force_resume = 1; 1845 } 1846 1847 return 0; 1848 1849 err: 1850 pm_runtime_enable(dev); 1851 return ret; 1852 } 1853 EXPORT_SYMBOL_GPL(pm_runtime_force_suspend); 1854 1855 /** 1856 * pm_runtime_force_resume - Force a device into resume state if needed. 1857 * @dev: Device to resume. 1858 * 1859 * Prior invoking this function we expect the user to have brought the device 1860 * into low power state by a call to pm_runtime_force_suspend(). Here we reverse 1861 * those actions and bring the device into full power, if it is expected to be 1862 * used on system resume. In the other case, we defer the resume to be managed 1863 * via runtime PM. 1864 * 1865 * Typically this function may be invoked from a system resume callback. 1866 */ 1867 int pm_runtime_force_resume(struct device *dev) 1868 { 1869 int (*callback)(struct device *); 1870 int ret = 0; 1871 1872 if (!pm_runtime_status_suspended(dev) || !dev->power.needs_force_resume) 1873 goto out; 1874 1875 /* 1876 * The value of the parent's children counter is correct already, so 1877 * just update the status of the device. 1878 */ 1879 __update_runtime_status(dev, RPM_ACTIVE); 1880 1881 callback = RPM_GET_CALLBACK(dev, runtime_resume); 1882 1883 ret = callback ? callback(dev) : 0; 1884 if (ret) { 1885 pm_runtime_set_suspended(dev); 1886 goto out; 1887 } 1888 1889 pm_runtime_mark_last_busy(dev); 1890 out: 1891 dev->power.needs_force_resume = 0; 1892 pm_runtime_enable(dev); 1893 return ret; 1894 } 1895 EXPORT_SYMBOL_GPL(pm_runtime_force_resume); 1896