179ffac85SChris Wilson /* 279ffac85SChris Wilson * SPDX-License-Identifier: MIT 379ffac85SChris Wilson * 479ffac85SChris Wilson * Copyright © 2019 Intel Corporation 579ffac85SChris Wilson */ 679ffac85SChris Wilson 7a70a9e99SChris Wilson #include <linux/suspend.h> 8a70a9e99SChris Wilson 979ffac85SChris Wilson #include "i915_drv.h" 10a2b4deadSChris Wilson #include "i915_globals.h" 11cb823ed9SChris Wilson #include "i915_params.h" 12dffa8febSChris Wilson #include "intel_context.h" 13092be382SChris Wilson #include "intel_engine_pm.h" 14cb823ed9SChris Wilson #include "intel_gt.h" 1579ffac85SChris Wilson #include "intel_gt_pm.h" 1666101975SChris Wilson #include "intel_gt_requests.h" 173e7abf81SAndi Shyti #include "intel_llc.h" 1879ffac85SChris Wilson #include "intel_pm.h" 19c1132367SAndi Shyti #include "intel_rc6.h" 203e7abf81SAndi Shyti #include "intel_rps.h" 2179ffac85SChris Wilson #include "intel_wakeref.h" 2279ffac85SChris Wilson 23d4033a9bSChris Wilson static void user_forcewake(struct intel_gt *gt, bool suspend) 24d4033a9bSChris Wilson { 25d4033a9bSChris Wilson int count = atomic_read(>->user_wakeref); 26d4033a9bSChris Wilson 27d4033a9bSChris Wilson /* Inside suspend/resume so single threaded, no races to worry about. */ 28d4033a9bSChris Wilson if (likely(!count)) 29d4033a9bSChris Wilson return; 30d4033a9bSChris Wilson 31d4033a9bSChris Wilson intel_gt_pm_get(gt); 32d4033a9bSChris Wilson if (suspend) { 33d4033a9bSChris Wilson GEM_BUG_ON(count > atomic_read(>->wakeref.count)); 34d4033a9bSChris Wilson atomic_sub(count, >->wakeref.count); 35d4033a9bSChris Wilson } else { 36d4033a9bSChris Wilson atomic_add(count, >->wakeref.count); 37d4033a9bSChris Wilson } 38d4033a9bSChris Wilson intel_gt_pm_put(gt); 39d4033a9bSChris Wilson } 40d4033a9bSChris Wilson 41c7302f20SChris Wilson static int __gt_unpark(struct intel_wakeref *wf) 4279ffac85SChris Wilson { 43cb823ed9SChris Wilson struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); 44cb823ed9SChris Wilson struct drm_i915_private *i915 = gt->i915; 4579ffac85SChris Wilson 46639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "\n"); 4779ffac85SChris Wilson 48a2b4deadSChris Wilson i915_globals_unpark(); 49a2b4deadSChris Wilson 5079ffac85SChris Wilson /* 5179ffac85SChris Wilson * It seems that the DMC likes to transition between the DC states a lot 5279ffac85SChris Wilson * when there are no connected displays (no active power domains) during 5379ffac85SChris Wilson * command submission. 5479ffac85SChris Wilson * 5579ffac85SChris Wilson * This activity has negative impact on the performance of the chip with 5679ffac85SChris Wilson * huge latencies observed in the interrupt handler and elsewhere. 5779ffac85SChris Wilson * 5879ffac85SChris Wilson * Work around it by grabbing a GT IRQ power domain whilst there is any 5979ffac85SChris Wilson * GT activity, preventing any DC state transitions. 6079ffac85SChris Wilson */ 61cb823ed9SChris Wilson gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); 62cb823ed9SChris Wilson GEM_BUG_ON(!gt->awake); 6379ffac85SChris Wilson 64730eaeb5SChris Wilson intel_rc6_unpark(>->rc6); 653e7abf81SAndi Shyti intel_rps_unpark(>->rps); 6679ffac85SChris Wilson i915_pmu_gt_unparked(i915); 6779ffac85SChris Wilson 6866101975SChris Wilson intel_gt_unpark_requests(gt); 6979ffac85SChris Wilson 7079ffac85SChris Wilson return 0; 7179ffac85SChris Wilson } 7279ffac85SChris Wilson 73c7302f20SChris Wilson static int __gt_park(struct intel_wakeref *wf) 7479ffac85SChris Wilson { 75ee236af8STvrtko Ursulin struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); 76ee236af8STvrtko Ursulin intel_wakeref_t wakeref = fetch_and_zero(>->awake); 77ee236af8STvrtko Ursulin struct drm_i915_private *i915 = gt->i915; 7879ffac85SChris Wilson 79639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "\n"); 8079ffac85SChris Wilson 8166101975SChris Wilson intel_gt_park_requests(gt); 8279ffac85SChris Wilson 8371e51ca8SChris Wilson i915_vma_parked(gt); 8479ffac85SChris Wilson i915_pmu_gt_parked(i915); 853e7abf81SAndi Shyti intel_rps_park(>->rps); 86730eaeb5SChris Wilson intel_rc6_park(>->rc6); 8779ffac85SChris Wilson 88c7302f20SChris Wilson /* Everything switched off, flush any residual interrupt just in case */ 89c7302f20SChris Wilson intel_synchronize_irq(i915); 90c7302f20SChris Wilson 9181ff52b7SChris Wilson /* Defer dropping the display power well for 100ms, it's slow! */ 9279ffac85SChris Wilson GEM_BUG_ON(!wakeref); 9381ff52b7SChris Wilson intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref); 9479ffac85SChris Wilson 95a2b4deadSChris Wilson i915_globals_park(); 96a2b4deadSChris Wilson 9779ffac85SChris Wilson return 0; 9879ffac85SChris Wilson } 9979ffac85SChris Wilson 100c7302f20SChris Wilson static const struct intel_wakeref_ops wf_ops = { 101c7302f20SChris Wilson .get = __gt_unpark, 102c7302f20SChris Wilson .put = __gt_park, 103c7302f20SChris Wilson }; 10479ffac85SChris Wilson 10599f2eb96STvrtko Ursulin void intel_gt_pm_init_early(struct intel_gt *gt) 10679ffac85SChris Wilson { 107cd6a8513SChris Wilson intel_wakeref_init(>->wakeref, gt->uncore->rpm, &wf_ops); 10879ffac85SChris Wilson } 10979ffac85SChris Wilson 110c1132367SAndi Shyti void intel_gt_pm_init(struct intel_gt *gt) 111c1132367SAndi Shyti { 112c1132367SAndi Shyti /* 113c1132367SAndi Shyti * Enabling power-management should be "self-healing". If we cannot 114c1132367SAndi Shyti * enable a feature, simply leave it disabled with a notice to the 115c1132367SAndi Shyti * user. 116c1132367SAndi Shyti */ 117c1132367SAndi Shyti intel_rc6_init(>->rc6); 1183e7abf81SAndi Shyti intel_rps_init(>->rps); 119c1132367SAndi Shyti } 120c1132367SAndi Shyti 121cb823ed9SChris Wilson static bool reset_engines(struct intel_gt *gt) 12279ffac85SChris Wilson { 123cb823ed9SChris Wilson if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display) 12479ffac85SChris Wilson return false; 12579ffac85SChris Wilson 126cb823ed9SChris Wilson return __intel_gt_reset(gt, ALL_ENGINES) == 0; 12779ffac85SChris Wilson } 12879ffac85SChris Wilson 12979ffac85SChris Wilson /** 13079ffac85SChris Wilson * intel_gt_sanitize: called after the GPU has lost power 1310c91621cSChris Wilson * @gt: the i915 GT container 13279ffac85SChris Wilson * @force: ignore a failed reset and sanitize engine state anyway 13379ffac85SChris Wilson * 13479ffac85SChris Wilson * Anytime we reset the GPU, either with an explicit GPU reset or through a 13579ffac85SChris Wilson * PCI power cycle, the GPU loses state and we must reset our state tracking 13679ffac85SChris Wilson * to match. Note that calling intel_gt_sanitize() if the GPU has not 13779ffac85SChris Wilson * been reset results in much confusion! 13879ffac85SChris Wilson */ 1390c91621cSChris Wilson void intel_gt_sanitize(struct intel_gt *gt, bool force) 14079ffac85SChris Wilson { 14179ffac85SChris Wilson struct intel_engine_cs *engine; 14279ffac85SChris Wilson enum intel_engine_id id; 143fd6fe087SChris Wilson intel_wakeref_t wakeref; 14479ffac85SChris Wilson 145639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "force:%s", yesno(force)); 146fd6fe087SChris Wilson 147fd6fe087SChris Wilson /* Use a raw wakeref to avoid calling intel_display_power_get early */ 148fd6fe087SChris Wilson wakeref = intel_runtime_pm_get(gt->uncore->rpm); 149fd6fe087SChris Wilson intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); 150fd6fe087SChris Wilson 151fd6fe087SChris Wilson /* 152fd6fe087SChris Wilson * As we have just resumed the machine and woken the device up from 153fd6fe087SChris Wilson * deep PCI sleep (presumably D3_cold), assume the HW has been reset 154fd6fe087SChris Wilson * back to defaults, recovering from whatever wedged state we left it 155fd6fe087SChris Wilson * in and so worth trying to use the device once more. 156fd6fe087SChris Wilson */ 157fd6fe087SChris Wilson if (intel_gt_is_wedged(gt)) 158fd6fe087SChris Wilson intel_gt_unset_wedged(gt); 15979ffac85SChris Wilson 160de6a2634SDaniele Ceraolo Spurio intel_uc_sanitize(>->uc); 161de6a2634SDaniele Ceraolo Spurio 1625d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) 1633c00660dSChris Wilson if (engine->reset.prepare) 1643c00660dSChris Wilson engine->reset.prepare(engine); 16579ffac85SChris Wilson 166fd6fe087SChris Wilson intel_uc_reset_prepare(>->uc); 167fd6fe087SChris Wilson 1683c00660dSChris Wilson if (reset_engines(gt) || force) { 1695d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) 170cb823ed9SChris Wilson __intel_engine_reset(engine, false); 17179ffac85SChris Wilson } 17279ffac85SChris Wilson 1735d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) 1743c00660dSChris Wilson if (engine->reset.finish) 1753c00660dSChris Wilson engine->reset.finish(engine); 176fd6fe087SChris Wilson 177fd6fe087SChris Wilson intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); 178fd6fe087SChris Wilson intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1793c00660dSChris Wilson } 1803c00660dSChris Wilson 181c1132367SAndi Shyti void intel_gt_pm_fini(struct intel_gt *gt) 182c1132367SAndi Shyti { 183c1132367SAndi Shyti intel_rc6_fini(>->rc6); 184c1132367SAndi Shyti } 185c1132367SAndi Shyti 186092be382SChris Wilson int intel_gt_resume(struct intel_gt *gt) 18779ffac85SChris Wilson { 18879ffac85SChris Wilson struct intel_engine_cs *engine; 18979ffac85SChris Wilson enum intel_engine_id id; 190092be382SChris Wilson int err = 0; 19179ffac85SChris Wilson 192639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "\n"); 193fd6fe087SChris Wilson 19479ffac85SChris Wilson /* 19579ffac85SChris Wilson * After resume, we may need to poke into the pinned kernel 19679ffac85SChris Wilson * contexts to paper over any damage caused by the sudden suspend. 19779ffac85SChris Wilson * Only the kernel contexts should remain pinned over suspend, 19879ffac85SChris Wilson * allowing us to fixup the user contexts on their first pin. 19979ffac85SChris Wilson */ 200092be382SChris Wilson intel_gt_pm_get(gt); 2013e7abf81SAndi Shyti 202c1132367SAndi Shyti intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); 203c1132367SAndi Shyti intel_rc6_sanitize(>->rc6); 204c1132367SAndi Shyti 2053e7abf81SAndi Shyti intel_rps_enable(>->rps); 2063e7abf81SAndi Shyti intel_llc_enable(>->llc); 2073e7abf81SAndi Shyti 2085d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) { 20979ffac85SChris Wilson struct intel_context *ce; 21079ffac85SChris Wilson 211092be382SChris Wilson intel_engine_pm_get(engine); 212092be382SChris Wilson 21379ffac85SChris Wilson ce = engine->kernel_context; 214dffa8febSChris Wilson if (ce) { 215dffa8febSChris Wilson GEM_BUG_ON(!intel_context_is_pinned(ce)); 21679ffac85SChris Wilson ce->ops->reset(ce); 217dffa8febSChris Wilson } 21879ffac85SChris Wilson 219092be382SChris Wilson engine->serial++; /* kernel context lost */ 220092be382SChris Wilson err = engine->resume(engine); 221092be382SChris Wilson 222092be382SChris Wilson intel_engine_pm_put(engine); 223092be382SChris Wilson if (err) { 224092be382SChris Wilson dev_err(gt->i915->drm.dev, 225092be382SChris Wilson "Failed to restart %s (%d)\n", 226092be382SChris Wilson engine->name, err); 227092be382SChris Wilson break; 22879ffac85SChris Wilson } 22979ffac85SChris Wilson } 230c1132367SAndi Shyti 231c1132367SAndi Shyti intel_rc6_enable(>->rc6); 232fd6fe087SChris Wilson 233fd6fe087SChris Wilson intel_uc_resume(>->uc); 234fd6fe087SChris Wilson 235d4033a9bSChris Wilson user_forcewake(gt, false); 236d4033a9bSChris Wilson 237c1132367SAndi Shyti intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); 238092be382SChris Wilson intel_gt_pm_put(gt); 239092be382SChris Wilson 240092be382SChris Wilson return err; 241092be382SChris Wilson } 2429dfe3459SDaniele Ceraolo Spurio 243a70a9e99SChris Wilson static void wait_for_suspend(struct intel_gt *gt) 244c1132367SAndi Shyti { 245a70a9e99SChris Wilson if (!intel_gt_pm_is_awake(gt)) 246a70a9e99SChris Wilson return; 247a70a9e99SChris Wilson 24866101975SChris Wilson if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) { 249c1132367SAndi Shyti /* 250c1132367SAndi Shyti * Forcibly cancel outstanding work and leave 251c1132367SAndi Shyti * the gpu quiet. 252c1132367SAndi Shyti */ 253c1132367SAndi Shyti intel_gt_set_wedged(gt); 2540cdfdf6fSChris Wilson intel_gt_retire_requests(gt); 255c1132367SAndi Shyti } 256c1132367SAndi Shyti 257c1132367SAndi Shyti intel_gt_pm_wait_for_idle(gt); 258c1132367SAndi Shyti } 259c1132367SAndi Shyti 260a70a9e99SChris Wilson void intel_gt_suspend_prepare(struct intel_gt *gt) 261a70a9e99SChris Wilson { 262a70a9e99SChris Wilson user_forcewake(gt, true); 263a70a9e99SChris Wilson wait_for_suspend(gt); 264a70a9e99SChris Wilson 265a70a9e99SChris Wilson intel_uc_suspend(>->uc); 266a70a9e99SChris Wilson } 267a70a9e99SChris Wilson 268a70a9e99SChris Wilson static suspend_state_t pm_suspend_target(void) 269a70a9e99SChris Wilson { 270e435c608SChris Wilson #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP) 271a70a9e99SChris Wilson return pm_suspend_target_state; 272a70a9e99SChris Wilson #else 273a70a9e99SChris Wilson return PM_SUSPEND_TO_IDLE; 274a70a9e99SChris Wilson #endif 275a70a9e99SChris Wilson } 276a70a9e99SChris Wilson 277a70a9e99SChris Wilson void intel_gt_suspend_late(struct intel_gt *gt) 278c1132367SAndi Shyti { 279c1132367SAndi Shyti intel_wakeref_t wakeref; 280c1132367SAndi Shyti 281c1132367SAndi Shyti /* We expect to be idle already; but also want to be independent */ 282a70a9e99SChris Wilson wait_for_suspend(gt); 283c1132367SAndi Shyti 284a70a9e99SChris Wilson /* 285a70a9e99SChris Wilson * On disabling the device, we want to turn off HW access to memory 286a70a9e99SChris Wilson * that we no longer own. 287a70a9e99SChris Wilson * 288a70a9e99SChris Wilson * However, not all suspend-states disable the device. S0 (s2idle) 289a70a9e99SChris Wilson * is effectively runtime-suspend, the device is left powered on 290a70a9e99SChris Wilson * but needs to be put into a low power state. We need to keep 291a70a9e99SChris Wilson * powermanagement enabled, but we also retain system state and so 292a70a9e99SChris Wilson * it remains safe to keep on using our allocated memory. 293a70a9e99SChris Wilson */ 294a70a9e99SChris Wilson if (pm_suspend_target() == PM_SUSPEND_TO_IDLE) 295a70a9e99SChris Wilson return; 296fd6fe087SChris Wilson 2973e7abf81SAndi Shyti with_intel_runtime_pm(gt->uncore->rpm, wakeref) { 2983e7abf81SAndi Shyti intel_rps_disable(>->rps); 299c1132367SAndi Shyti intel_rc6_disable(>->rc6); 3003e7abf81SAndi Shyti intel_llc_disable(>->llc); 3013e7abf81SAndi Shyti } 302fd6fe087SChris Wilson 303fd6fe087SChris Wilson intel_gt_sanitize(gt, false); 304fd6fe087SChris Wilson 305639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "\n"); 306c1132367SAndi Shyti } 307c1132367SAndi Shyti 3089dfe3459SDaniele Ceraolo Spurio void intel_gt_runtime_suspend(struct intel_gt *gt) 3099dfe3459SDaniele Ceraolo Spurio { 3109dfe3459SDaniele Ceraolo Spurio intel_uc_runtime_suspend(>->uc); 311fd6fe087SChris Wilson 312639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "\n"); 3139dfe3459SDaniele Ceraolo Spurio } 3149dfe3459SDaniele Ceraolo Spurio 3159dfe3459SDaniele Ceraolo Spurio int intel_gt_runtime_resume(struct intel_gt *gt) 3169dfe3459SDaniele Ceraolo Spurio { 317639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "\n"); 3189dfe3459SDaniele Ceraolo Spurio intel_gt_init_swizzling(gt); 3199dfe3459SDaniele Ceraolo Spurio 3209dfe3459SDaniele Ceraolo Spurio return intel_uc_runtime_resume(>->uc); 3219dfe3459SDaniele Ceraolo Spurio } 322c1132367SAndi Shyti 323c1132367SAndi Shyti #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 324c1132367SAndi Shyti #include "selftest_gt_pm.c" 325c1132367SAndi Shyti #endif 326