1 /* 2 * Copyright © 2012-2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eugeni Dodonov <eugeni.dodonov@intel.com> 25 * Daniel Vetter <daniel.vetter@ffwll.ch> 26 * 27 */ 28 29 #include <linux/pm_runtime.h> 30 31 #include <drm/drm_print.h> 32 33 #include "i915_drv.h" 34 #include "i915_trace.h" 35 36 /** 37 * DOC: runtime pm 38 * 39 * The i915 driver supports dynamic enabling and disabling of entire hardware 40 * blocks at runtime. This is especially important on the display side where 41 * software is supposed to control many power gates manually on recent hardware, 42 * since on the GT side a lot of the power management is done by the hardware. 43 * But even there some manual control at the device level is required. 44 * 45 * Since i915 supports a diverse set of platforms with a unified codebase and 46 * hardware engineers just love to shuffle functionality around between power 47 * domains there's a sizeable amount of indirection required. This file provides 48 * generic functions to the driver for grabbing and releasing references for 49 * abstract power domains. It then maps those to the actual power wells 50 * present for a given platform. 51 */ 52 53 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) 54 55 #include <linux/sort.h> 56 57 #define STACKDEPTH 8 58 59 static noinline depot_stack_handle_t __save_depot_stack(void) 60 { 61 unsigned long entries[STACKDEPTH]; 62 unsigned int n; 63 64 n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 65 return stack_depot_save(entries, n, GFP_NOWAIT | __GFP_NOWARN); 66 } 67 68 static void __print_depot_stack(depot_stack_handle_t stack, 69 char *buf, int sz, int indent) 70 { 71 unsigned long *entries; 72 unsigned int nr_entries; 73 74 nr_entries = stack_depot_fetch(stack, &entries); 75 stack_trace_snprint(buf, sz, entries, nr_entries, indent); 76 } 77 78 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) 79 { 80 spin_lock_init(&rpm->debug.lock); 81 } 82 83 static noinline depot_stack_handle_t 84 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) 85 { 86 depot_stack_handle_t stack, *stacks; 87 unsigned long flags; 88 89 if (!rpm->available) 90 return -1; 91 92 stack = __save_depot_stack(); 93 if (!stack) 94 return -1; 95 96 spin_lock_irqsave(&rpm->debug.lock, flags); 97 98 if (!rpm->debug.count) 99 rpm->debug.last_acquire = stack; 100 101 stacks = krealloc(rpm->debug.owners, 102 (rpm->debug.count + 1) * sizeof(*stacks), 103 GFP_NOWAIT | __GFP_NOWARN); 104 if (stacks) { 105 stacks[rpm->debug.count++] = stack; 106 rpm->debug.owners = stacks; 107 } else { 108 stack = -1; 109 } 110 111 spin_unlock_irqrestore(&rpm->debug.lock, flags); 112 113 return stack; 114 } 115 116 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm, 117 depot_stack_handle_t stack) 118 { 119 unsigned long flags, n; 120 bool found = false; 121 122 if (unlikely(stack == -1)) 123 return; 124 125 spin_lock_irqsave(&rpm->debug.lock, flags); 126 for (n = rpm->debug.count; n--; ) { 127 if (rpm->debug.owners[n] == stack) { 128 memmove(rpm->debug.owners + n, 129 rpm->debug.owners + n + 1, 130 (--rpm->debug.count - n) * sizeof(stack)); 131 found = true; 132 break; 133 } 134 } 135 spin_unlock_irqrestore(&rpm->debug.lock, flags); 136 137 if (WARN(!found, 138 "Unmatched wakeref (tracking %lu), count %u\n", 139 rpm->debug.count, atomic_read(&rpm->wakeref_count))) { 140 char *buf; 141 142 buf = kmalloc(PAGE_SIZE, GFP_NOWAIT | __GFP_NOWARN); 143 if (!buf) 144 return; 145 146 __print_depot_stack(stack, buf, PAGE_SIZE, 2); 147 DRM_DEBUG_DRIVER("wakeref %x from\n%s", stack, buf); 148 149 stack = READ_ONCE(rpm->debug.last_release); 150 if (stack) { 151 __print_depot_stack(stack, buf, PAGE_SIZE, 2); 152 DRM_DEBUG_DRIVER("wakeref last released at\n%s", buf); 153 } 154 155 kfree(buf); 156 } 157 } 158 159 static int cmphandle(const void *_a, const void *_b) 160 { 161 const depot_stack_handle_t * const a = _a, * const b = _b; 162 163 if (*a < *b) 164 return -1; 165 else if (*a > *b) 166 return 1; 167 else 168 return 0; 169 } 170 171 static void 172 __print_intel_runtime_pm_wakeref(struct drm_printer *p, 173 const struct intel_runtime_pm_debug *dbg) 174 { 175 unsigned long i; 176 char *buf; 177 178 buf = kmalloc(PAGE_SIZE, GFP_NOWAIT | __GFP_NOWARN); 179 if (!buf) 180 return; 181 182 if (dbg->last_acquire) { 183 __print_depot_stack(dbg->last_acquire, buf, PAGE_SIZE, 2); 184 drm_printf(p, "Wakeref last acquired:\n%s", buf); 185 } 186 187 if (dbg->last_release) { 188 __print_depot_stack(dbg->last_release, buf, PAGE_SIZE, 2); 189 drm_printf(p, "Wakeref last released:\n%s", buf); 190 } 191 192 drm_printf(p, "Wakeref count: %lu\n", dbg->count); 193 194 sort(dbg->owners, dbg->count, sizeof(*dbg->owners), cmphandle, NULL); 195 196 for (i = 0; i < dbg->count; i++) { 197 depot_stack_handle_t stack = dbg->owners[i]; 198 unsigned long rep; 199 200 rep = 1; 201 while (i + 1 < dbg->count && dbg->owners[i + 1] == stack) 202 rep++, i++; 203 __print_depot_stack(stack, buf, PAGE_SIZE, 2); 204 drm_printf(p, "Wakeref x%lu taken at:\n%s", rep, buf); 205 } 206 207 kfree(buf); 208 } 209 210 static noinline void 211 __untrack_all_wakerefs(struct intel_runtime_pm_debug *debug, 212 struct intel_runtime_pm_debug *saved) 213 { 214 *saved = *debug; 215 216 debug->owners = NULL; 217 debug->count = 0; 218 debug->last_release = __save_depot_stack(); 219 } 220 221 static void 222 dump_and_free_wakeref_tracking(struct intel_runtime_pm_debug *debug) 223 { 224 if (debug->count) { 225 struct drm_printer p = drm_debug_printer("i915"); 226 227 __print_intel_runtime_pm_wakeref(&p, debug); 228 } 229 230 kfree(debug->owners); 231 } 232 233 static noinline void 234 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm) 235 { 236 struct intel_runtime_pm_debug dbg = {}; 237 unsigned long flags; 238 239 if (!atomic_dec_and_lock_irqsave(&rpm->wakeref_count, 240 &rpm->debug.lock, 241 flags)) 242 return; 243 244 __untrack_all_wakerefs(&rpm->debug, &dbg); 245 spin_unlock_irqrestore(&rpm->debug.lock, flags); 246 247 dump_and_free_wakeref_tracking(&dbg); 248 } 249 250 static noinline void 251 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm) 252 { 253 struct intel_runtime_pm_debug dbg = {}; 254 unsigned long flags; 255 256 spin_lock_irqsave(&rpm->debug.lock, flags); 257 __untrack_all_wakerefs(&rpm->debug, &dbg); 258 spin_unlock_irqrestore(&rpm->debug.lock, flags); 259 260 dump_and_free_wakeref_tracking(&dbg); 261 } 262 263 void print_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm, 264 struct drm_printer *p) 265 { 266 struct intel_runtime_pm_debug dbg = {}; 267 268 do { 269 unsigned long alloc = dbg.count; 270 depot_stack_handle_t *s; 271 272 spin_lock_irq(&rpm->debug.lock); 273 dbg.count = rpm->debug.count; 274 if (dbg.count <= alloc) { 275 memcpy(dbg.owners, 276 rpm->debug.owners, 277 dbg.count * sizeof(*s)); 278 } 279 dbg.last_acquire = rpm->debug.last_acquire; 280 dbg.last_release = rpm->debug.last_release; 281 spin_unlock_irq(&rpm->debug.lock); 282 if (dbg.count <= alloc) 283 break; 284 285 s = krealloc(dbg.owners, 286 dbg.count * sizeof(*s), 287 GFP_NOWAIT | __GFP_NOWARN); 288 if (!s) 289 goto out; 290 291 dbg.owners = s; 292 } while (1); 293 294 __print_intel_runtime_pm_wakeref(p, &dbg); 295 296 out: 297 kfree(dbg.owners); 298 } 299 300 #else 301 302 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) 303 { 304 } 305 306 static depot_stack_handle_t 307 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm) 308 { 309 return -1; 310 } 311 312 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm, 313 intel_wakeref_t wref) 314 { 315 } 316 317 static void 318 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm) 319 { 320 atomic_dec(&rpm->wakeref_count); 321 } 322 323 static void 324 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm) 325 { 326 } 327 328 #endif 329 330 static void 331 intel_runtime_pm_acquire(struct intel_runtime_pm *rpm, bool wakelock) 332 { 333 if (wakelock) { 334 atomic_add(1 + INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count); 335 assert_rpm_wakelock_held(rpm); 336 } else { 337 atomic_inc(&rpm->wakeref_count); 338 assert_rpm_raw_wakeref_held(rpm); 339 } 340 } 341 342 static void 343 intel_runtime_pm_release(struct intel_runtime_pm *rpm, int wakelock) 344 { 345 if (wakelock) { 346 assert_rpm_wakelock_held(rpm); 347 atomic_sub(INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count); 348 } else { 349 assert_rpm_raw_wakeref_held(rpm); 350 } 351 352 __intel_wakeref_dec_and_check_tracking(rpm); 353 } 354 355 static intel_wakeref_t __intel_runtime_pm_get(struct intel_runtime_pm *rpm, 356 bool wakelock) 357 { 358 int ret; 359 360 ret = pm_runtime_get_sync(rpm->kdev); 361 WARN_ONCE(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret); 362 363 intel_runtime_pm_acquire(rpm, wakelock); 364 365 return track_intel_runtime_pm_wakeref(rpm); 366 } 367 368 /** 369 * intel_runtime_pm_get_raw - grab a raw runtime pm reference 370 * @rpm: the intel_runtime_pm structure 371 * 372 * This is the unlocked version of intel_display_power_is_enabled() and should 373 * only be used from error capture and recovery code where deadlocks are 374 * possible. 375 * This function grabs a device-level runtime pm reference (mostly used for 376 * asynchronous PM management from display code) and ensures that it is powered 377 * up. Raw references are not considered during wakelock assert checks. 378 * 379 * Any runtime pm reference obtained by this function must have a symmetric 380 * call to intel_runtime_pm_put_raw() to release the reference again. 381 * 382 * Returns: the wakeref cookie to pass to intel_runtime_pm_put_raw(), evaluates 383 * as True if the wakeref was acquired, or False otherwise. 384 */ 385 intel_wakeref_t intel_runtime_pm_get_raw(struct intel_runtime_pm *rpm) 386 { 387 return __intel_runtime_pm_get(rpm, false); 388 } 389 390 /** 391 * intel_runtime_pm_get - grab a runtime pm reference 392 * @rpm: the intel_runtime_pm structure 393 * 394 * This function grabs a device-level runtime pm reference (mostly used for GEM 395 * code to ensure the GTT or GT is on) and ensures that it is powered up. 396 * 397 * Any runtime pm reference obtained by this function must have a symmetric 398 * call to intel_runtime_pm_put() to release the reference again. 399 * 400 * Returns: the wakeref cookie to pass to intel_runtime_pm_put() 401 */ 402 intel_wakeref_t intel_runtime_pm_get(struct intel_runtime_pm *rpm) 403 { 404 return __intel_runtime_pm_get(rpm, true); 405 } 406 407 /** 408 * intel_runtime_pm_get_if_in_use - grab a runtime pm reference if device in use 409 * @rpm: the intel_runtime_pm structure 410 * 411 * This function grabs a device-level runtime pm reference if the device is 412 * already in use and ensures that it is powered up. It is illegal to try 413 * and access the HW should intel_runtime_pm_get_if_in_use() report failure. 414 * 415 * Any runtime pm reference obtained by this function must have a symmetric 416 * call to intel_runtime_pm_put() to release the reference again. 417 * 418 * Returns: the wakeref cookie to pass to intel_runtime_pm_put(), evaluates 419 * as True if the wakeref was acquired, or False otherwise. 420 */ 421 intel_wakeref_t intel_runtime_pm_get_if_in_use(struct intel_runtime_pm *rpm) 422 { 423 if (IS_ENABLED(CONFIG_PM)) { 424 /* 425 * In cases runtime PM is disabled by the RPM core and we get 426 * an -EINVAL return value we are not supposed to call this 427 * function, since the power state is undefined. This applies 428 * atm to the late/early system suspend/resume handlers. 429 */ 430 if (pm_runtime_get_if_in_use(rpm->kdev) <= 0) 431 return 0; 432 } 433 434 intel_runtime_pm_acquire(rpm, true); 435 436 return track_intel_runtime_pm_wakeref(rpm); 437 } 438 439 /** 440 * intel_runtime_pm_get_noresume - grab a runtime pm reference 441 * @rpm: the intel_runtime_pm structure 442 * 443 * This function grabs a device-level runtime pm reference (mostly used for GEM 444 * code to ensure the GTT or GT is on). 445 * 446 * It will _not_ power up the device but instead only check that it's powered 447 * on. Therefore it is only valid to call this functions from contexts where 448 * the device is known to be powered up and where trying to power it up would 449 * result in hilarity and deadlocks. That pretty much means only the system 450 * suspend/resume code where this is used to grab runtime pm references for 451 * delayed setup down in work items. 452 * 453 * Any runtime pm reference obtained by this function must have a symmetric 454 * call to intel_runtime_pm_put() to release the reference again. 455 * 456 * Returns: the wakeref cookie to pass to intel_runtime_pm_put() 457 */ 458 intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm) 459 { 460 assert_rpm_wakelock_held(rpm); 461 pm_runtime_get_noresume(rpm->kdev); 462 463 intel_runtime_pm_acquire(rpm, true); 464 465 return track_intel_runtime_pm_wakeref(rpm); 466 } 467 468 static void __intel_runtime_pm_put(struct intel_runtime_pm *rpm, 469 intel_wakeref_t wref, 470 bool wakelock) 471 { 472 struct device *kdev = rpm->kdev; 473 474 untrack_intel_runtime_pm_wakeref(rpm, wref); 475 476 intel_runtime_pm_release(rpm, wakelock); 477 478 pm_runtime_mark_last_busy(kdev); 479 pm_runtime_put_autosuspend(kdev); 480 } 481 482 /** 483 * intel_runtime_pm_put_raw - release a raw runtime pm reference 484 * @rpm: the intel_runtime_pm structure 485 * @wref: wakeref acquired for the reference that is being released 486 * 487 * This function drops the device-level runtime pm reference obtained by 488 * intel_runtime_pm_get_raw() and might power down the corresponding 489 * hardware block right away if this is the last reference. 490 */ 491 void 492 intel_runtime_pm_put_raw(struct intel_runtime_pm *rpm, intel_wakeref_t wref) 493 { 494 __intel_runtime_pm_put(rpm, wref, false); 495 } 496 497 /** 498 * intel_runtime_pm_put_unchecked - release an unchecked runtime pm reference 499 * @rpm: the intel_runtime_pm structure 500 * 501 * This function drops the device-level runtime pm reference obtained by 502 * intel_runtime_pm_get() and might power down the corresponding 503 * hardware block right away if this is the last reference. 504 * 505 * This function exists only for historical reasons and should be avoided in 506 * new code, as the correctness of its use cannot be checked. Always use 507 * intel_runtime_pm_put() instead. 508 */ 509 void intel_runtime_pm_put_unchecked(struct intel_runtime_pm *rpm) 510 { 511 __intel_runtime_pm_put(rpm, -1, true); 512 } 513 514 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) 515 /** 516 * intel_runtime_pm_put - release a runtime pm reference 517 * @rpm: the intel_runtime_pm structure 518 * @wref: wakeref acquired for the reference that is being released 519 * 520 * This function drops the device-level runtime pm reference obtained by 521 * intel_runtime_pm_get() and might power down the corresponding 522 * hardware block right away if this is the last reference. 523 */ 524 void intel_runtime_pm_put(struct intel_runtime_pm *rpm, intel_wakeref_t wref) 525 { 526 __intel_runtime_pm_put(rpm, wref, true); 527 } 528 #endif 529 530 /** 531 * intel_runtime_pm_enable - enable runtime pm 532 * @rpm: the intel_runtime_pm structure 533 * 534 * This function enables runtime pm at the end of the driver load sequence. 535 * 536 * Note that this function does currently not enable runtime pm for the 537 * subordinate display power domains. That is done by 538 * intel_power_domains_enable(). 539 */ 540 void intel_runtime_pm_enable(struct intel_runtime_pm *rpm) 541 { 542 struct device *kdev = rpm->kdev; 543 544 /* 545 * Disable the system suspend direct complete optimization, which can 546 * leave the device suspended skipping the driver's suspend handlers 547 * if the device was already runtime suspended. This is needed due to 548 * the difference in our runtime and system suspend sequence and 549 * becaue the HDA driver may require us to enable the audio power 550 * domain during system suspend. 551 */ 552 dev_pm_set_driver_flags(kdev, DPM_FLAG_NO_DIRECT_COMPLETE); 553 554 pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */ 555 pm_runtime_mark_last_busy(kdev); 556 557 /* 558 * Take a permanent reference to disable the RPM functionality and drop 559 * it only when unloading the driver. Use the low level get/put helpers, 560 * so the driver's own RPM reference tracking asserts also work on 561 * platforms without RPM support. 562 */ 563 if (!rpm->available) { 564 int ret; 565 566 pm_runtime_dont_use_autosuspend(kdev); 567 ret = pm_runtime_get_sync(kdev); 568 WARN(ret < 0, "pm_runtime_get_sync() failed: %d\n", ret); 569 } else { 570 pm_runtime_use_autosuspend(kdev); 571 } 572 573 /* 574 * The core calls the driver load handler with an RPM reference held. 575 * We drop that here and will reacquire it during unloading in 576 * intel_power_domains_fini(). 577 */ 578 pm_runtime_put_autosuspend(kdev); 579 } 580 581 void intel_runtime_pm_disable(struct intel_runtime_pm *rpm) 582 { 583 struct device *kdev = rpm->kdev; 584 585 /* Transfer rpm ownership back to core */ 586 WARN(pm_runtime_get_sync(kdev) < 0, 587 "Failed to pass rpm ownership back to core\n"); 588 589 pm_runtime_dont_use_autosuspend(kdev); 590 591 if (!rpm->available) 592 pm_runtime_put(kdev); 593 } 594 595 void intel_runtime_pm_driver_release(struct intel_runtime_pm *rpm) 596 { 597 int count = atomic_read(&rpm->wakeref_count); 598 599 WARN(count, 600 "i915 raw-wakerefs=%d wakelocks=%d on cleanup\n", 601 intel_rpm_raw_wakeref_count(count), 602 intel_rpm_wakelock_count(count)); 603 604 untrack_all_intel_runtime_pm_wakerefs(rpm); 605 } 606 607 void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm) 608 { 609 struct drm_i915_private *i915 = 610 container_of(rpm, struct drm_i915_private, runtime_pm); 611 struct pci_dev *pdev = i915->drm.pdev; 612 struct device *kdev = &pdev->dev; 613 614 rpm->kdev = kdev; 615 rpm->available = HAS_RUNTIME_PM(i915); 616 617 init_intel_runtime_pm_wakeref(rpm); 618 } 619