179ffac85SChris Wilson /* 279ffac85SChris Wilson * SPDX-License-Identifier: MIT 379ffac85SChris Wilson * 479ffac85SChris Wilson * Copyright © 2019 Intel Corporation 579ffac85SChris Wilson */ 679ffac85SChris Wilson 7a70a9e99SChris Wilson #include <linux/suspend.h> 8a70a9e99SChris Wilson 979ffac85SChris Wilson #include "i915_drv.h" 10a2b4deadSChris Wilson #include "i915_globals.h" 11cb823ed9SChris Wilson #include "i915_params.h" 12dffa8febSChris Wilson #include "intel_context.h" 13092be382SChris Wilson #include "intel_engine_pm.h" 14cb823ed9SChris Wilson #include "intel_gt.h" 159c878557SChris Wilson #include "intel_gt_clock_utils.h" 1679ffac85SChris Wilson #include "intel_gt_pm.h" 1766101975SChris Wilson #include "intel_gt_requests.h" 183e7abf81SAndi Shyti #include "intel_llc.h" 1979ffac85SChris Wilson #include "intel_pm.h" 20c1132367SAndi Shyti #include "intel_rc6.h" 213e7abf81SAndi Shyti #include "intel_rps.h" 2279ffac85SChris Wilson #include "intel_wakeref.h" 2379ffac85SChris Wilson 24d4033a9bSChris Wilson static void user_forcewake(struct intel_gt *gt, bool suspend) 25d4033a9bSChris Wilson { 26d4033a9bSChris Wilson int count = atomic_read(>->user_wakeref); 27d4033a9bSChris Wilson 28d4033a9bSChris Wilson /* Inside suspend/resume so single threaded, no races to worry about. */ 29d4033a9bSChris Wilson if (likely(!count)) 30d4033a9bSChris Wilson return; 31d4033a9bSChris Wilson 32d4033a9bSChris Wilson intel_gt_pm_get(gt); 33d4033a9bSChris Wilson if (suspend) { 34d4033a9bSChris Wilson GEM_BUG_ON(count > atomic_read(>->wakeref.count)); 35d4033a9bSChris Wilson atomic_sub(count, >->wakeref.count); 36d4033a9bSChris Wilson } else { 37d4033a9bSChris Wilson atomic_add(count, >->wakeref.count); 38d4033a9bSChris Wilson } 39d4033a9bSChris Wilson intel_gt_pm_put(gt); 40d4033a9bSChris Wilson } 41d4033a9bSChris Wilson 42c7302f20SChris Wilson static int __gt_unpark(struct intel_wakeref *wf) 4379ffac85SChris Wilson { 44cb823ed9SChris Wilson struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); 45cb823ed9SChris Wilson struct drm_i915_private *i915 = gt->i915; 4679ffac85SChris Wilson 47639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "\n"); 4879ffac85SChris Wilson 49a2b4deadSChris Wilson i915_globals_unpark(); 50a2b4deadSChris Wilson 5179ffac85SChris Wilson /* 5279ffac85SChris Wilson * It seems that the DMC likes to transition between the DC states a lot 5379ffac85SChris Wilson * when there are no connected displays (no active power domains) during 5479ffac85SChris Wilson * command submission. 5579ffac85SChris Wilson * 5679ffac85SChris Wilson * This activity has negative impact on the performance of the chip with 5779ffac85SChris Wilson * huge latencies observed in the interrupt handler and elsewhere. 5879ffac85SChris Wilson * 5979ffac85SChris Wilson * Work around it by grabbing a GT IRQ power domain whilst there is any 6079ffac85SChris Wilson * GT activity, preventing any DC state transitions. 6179ffac85SChris Wilson */ 62cb823ed9SChris Wilson gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); 63cb823ed9SChris Wilson GEM_BUG_ON(!gt->awake); 6479ffac85SChris Wilson 65730eaeb5SChris Wilson intel_rc6_unpark(>->rc6); 663e7abf81SAndi Shyti intel_rps_unpark(>->rps); 6779ffac85SChris Wilson i915_pmu_gt_unparked(i915); 6879ffac85SChris Wilson 6966101975SChris Wilson intel_gt_unpark_requests(gt); 7079ffac85SChris Wilson 7179ffac85SChris Wilson return 0; 7279ffac85SChris Wilson } 7379ffac85SChris Wilson 74c7302f20SChris Wilson static int __gt_park(struct intel_wakeref *wf) 7579ffac85SChris Wilson { 76ee236af8STvrtko Ursulin struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); 77ee236af8STvrtko Ursulin intel_wakeref_t wakeref = fetch_and_zero(>->awake); 78ee236af8STvrtko Ursulin struct drm_i915_private *i915 = gt->i915; 7979ffac85SChris Wilson 80639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "\n"); 8179ffac85SChris Wilson 8266101975SChris Wilson intel_gt_park_requests(gt); 8379ffac85SChris Wilson 8471e51ca8SChris Wilson i915_vma_parked(gt); 8579ffac85SChris Wilson i915_pmu_gt_parked(i915); 863e7abf81SAndi Shyti intel_rps_park(>->rps); 87730eaeb5SChris Wilson intel_rc6_park(>->rc6); 8879ffac85SChris Wilson 89c7302f20SChris Wilson /* Everything switched off, flush any residual interrupt just in case */ 90c7302f20SChris Wilson intel_synchronize_irq(i915); 91c7302f20SChris Wilson 9281ff52b7SChris Wilson /* Defer dropping the display power well for 100ms, it's slow! */ 9379ffac85SChris Wilson GEM_BUG_ON(!wakeref); 9481ff52b7SChris Wilson intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref); 9579ffac85SChris Wilson 96a2b4deadSChris Wilson i915_globals_park(); 97a2b4deadSChris Wilson 9879ffac85SChris Wilson return 0; 9979ffac85SChris Wilson } 10079ffac85SChris Wilson 101c7302f20SChris Wilson static const struct intel_wakeref_ops wf_ops = { 102c7302f20SChris Wilson .get = __gt_unpark, 103c7302f20SChris Wilson .put = __gt_park, 104c7302f20SChris Wilson }; 10579ffac85SChris Wilson 10699f2eb96STvrtko Ursulin void intel_gt_pm_init_early(struct intel_gt *gt) 10779ffac85SChris Wilson { 108cd6a8513SChris Wilson intel_wakeref_init(>->wakeref, gt->uncore->rpm, &wf_ops); 10979ffac85SChris Wilson } 11079ffac85SChris Wilson 111c1132367SAndi Shyti void intel_gt_pm_init(struct intel_gt *gt) 112c1132367SAndi Shyti { 113c1132367SAndi Shyti /* 114c1132367SAndi Shyti * Enabling power-management should be "self-healing". If we cannot 115c1132367SAndi Shyti * enable a feature, simply leave it disabled with a notice to the 116c1132367SAndi Shyti * user. 117c1132367SAndi Shyti */ 118c1132367SAndi Shyti intel_rc6_init(>->rc6); 1193e7abf81SAndi Shyti intel_rps_init(>->rps); 120c1132367SAndi Shyti } 121c1132367SAndi Shyti 122cb823ed9SChris Wilson static bool reset_engines(struct intel_gt *gt) 12379ffac85SChris Wilson { 124cb823ed9SChris Wilson if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display) 12579ffac85SChris Wilson return false; 12679ffac85SChris Wilson 127cb823ed9SChris Wilson return __intel_gt_reset(gt, ALL_ENGINES) == 0; 12879ffac85SChris Wilson } 12979ffac85SChris Wilson 130d03b224fSChris Wilson static void gt_sanitize(struct intel_gt *gt, bool force) 13179ffac85SChris Wilson { 13279ffac85SChris Wilson struct intel_engine_cs *engine; 13379ffac85SChris Wilson enum intel_engine_id id; 134fd6fe087SChris Wilson intel_wakeref_t wakeref; 13579ffac85SChris Wilson 136639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "force:%s", yesno(force)); 137fd6fe087SChris Wilson 138fd6fe087SChris Wilson /* Use a raw wakeref to avoid calling intel_display_power_get early */ 139fd6fe087SChris Wilson wakeref = intel_runtime_pm_get(gt->uncore->rpm); 140fd6fe087SChris Wilson intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); 141fd6fe087SChris Wilson 1429c878557SChris Wilson intel_gt_check_clock_frequency(gt); 1439c878557SChris Wilson 144fd6fe087SChris Wilson /* 145fd6fe087SChris Wilson * As we have just resumed the machine and woken the device up from 146fd6fe087SChris Wilson * deep PCI sleep (presumably D3_cold), assume the HW has been reset 147fd6fe087SChris Wilson * back to defaults, recovering from whatever wedged state we left it 148fd6fe087SChris Wilson * in and so worth trying to use the device once more. 149fd6fe087SChris Wilson */ 150fd6fe087SChris Wilson if (intel_gt_is_wedged(gt)) 151fd6fe087SChris Wilson intel_gt_unset_wedged(gt); 15279ffac85SChris Wilson 153de6a2634SDaniele Ceraolo Spurio intel_uc_sanitize(>->uc); 154de6a2634SDaniele Ceraolo Spurio 1555d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) 1563c00660dSChris Wilson if (engine->reset.prepare) 1573c00660dSChris Wilson engine->reset.prepare(engine); 15879ffac85SChris Wilson 159fd6fe087SChris Wilson intel_uc_reset_prepare(>->uc); 160fd6fe087SChris Wilson 1614a0ca47aSChris Wilson for_each_engine(engine, gt, id) 1624a0ca47aSChris Wilson if (engine->sanitize) 1634a0ca47aSChris Wilson engine->sanitize(engine); 1644a0ca47aSChris Wilson 1653c00660dSChris Wilson if (reset_engines(gt) || force) { 1665d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) 167cb823ed9SChris Wilson __intel_engine_reset(engine, false); 16879ffac85SChris Wilson } 16979ffac85SChris Wilson 1705d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) 1713c00660dSChris Wilson if (engine->reset.finish) 1723c00660dSChris Wilson engine->reset.finish(engine); 173fd6fe087SChris Wilson 174389b7f00SChris Wilson intel_rps_sanitize(>->rps); 175389b7f00SChris Wilson 176fd6fe087SChris Wilson intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); 177fd6fe087SChris Wilson intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1783c00660dSChris Wilson } 1793c00660dSChris Wilson 180c1132367SAndi Shyti void intel_gt_pm_fini(struct intel_gt *gt) 181c1132367SAndi Shyti { 182c1132367SAndi Shyti intel_rc6_fini(>->rc6); 183c1132367SAndi Shyti } 184c1132367SAndi Shyti 185092be382SChris Wilson int intel_gt_resume(struct intel_gt *gt) 18679ffac85SChris Wilson { 18779ffac85SChris Wilson struct intel_engine_cs *engine; 18879ffac85SChris Wilson enum intel_engine_id id; 189cfe6b30fSChris Wilson int err; 19079ffac85SChris Wilson 1913f04bdceSMichał Winiarski err = intel_gt_has_unrecoverable_error(gt); 192d03b224fSChris Wilson if (err) 193d03b224fSChris Wilson return err; 194d03b224fSChris Wilson 195639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "\n"); 196fd6fe087SChris Wilson 19779ffac85SChris Wilson /* 19879ffac85SChris Wilson * After resume, we may need to poke into the pinned kernel 19979ffac85SChris Wilson * contexts to paper over any damage caused by the sudden suspend. 20079ffac85SChris Wilson * Only the kernel contexts should remain pinned over suspend, 20179ffac85SChris Wilson * allowing us to fixup the user contexts on their first pin. 20279ffac85SChris Wilson */ 2034243cd53SChris Wilson gt_sanitize(gt, true); 2044243cd53SChris Wilson 205092be382SChris Wilson intel_gt_pm_get(gt); 2063e7abf81SAndi Shyti 207c1132367SAndi Shyti intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); 208c1132367SAndi Shyti intel_rc6_sanitize(>->rc6); 20945b152f7SChris Wilson if (intel_gt_is_wedged(gt)) { 21045b152f7SChris Wilson err = -EIO; 21145b152f7SChris Wilson goto out_fw; 21245b152f7SChris Wilson } 213c1132367SAndi Shyti 214cfe6b30fSChris Wilson /* Only when the HW is re-initialised, can we replay the requests */ 215cfe6b30fSChris Wilson err = intel_gt_init_hw(gt); 216cfe6b30fSChris Wilson if (err) { 2176783ebdaSChris Wilson i915_probe_error(gt->i915, 218cfe6b30fSChris Wilson "Failed to initialize GPU, declaring it wedged!\n"); 219d03b224fSChris Wilson goto err_wedged; 220cfe6b30fSChris Wilson } 221cfe6b30fSChris Wilson 2223e7abf81SAndi Shyti intel_rps_enable(>->rps); 2233e7abf81SAndi Shyti intel_llc_enable(>->llc); 2243e7abf81SAndi Shyti 2255d904e3cSTvrtko Ursulin for_each_engine(engine, gt, id) { 226092be382SChris Wilson intel_engine_pm_get(engine); 227092be382SChris Wilson 228092be382SChris Wilson engine->serial++; /* kernel context lost */ 229faea1792SDaniele Ceraolo Spurio err = intel_engine_resume(engine); 230092be382SChris Wilson 231092be382SChris Wilson intel_engine_pm_put(engine); 232092be382SChris Wilson if (err) { 233dc483ba5SJani Nikula drm_err(>->i915->drm, 234092be382SChris Wilson "Failed to restart %s (%d)\n", 235092be382SChris Wilson engine->name, err); 236d03b224fSChris Wilson goto err_wedged; 23779ffac85SChris Wilson } 23879ffac85SChris Wilson } 239c1132367SAndi Shyti 240c1132367SAndi Shyti intel_rc6_enable(>->rc6); 241fd6fe087SChris Wilson 242fd6fe087SChris Wilson intel_uc_resume(>->uc); 243fd6fe087SChris Wilson 244d4033a9bSChris Wilson user_forcewake(gt, false); 245d4033a9bSChris Wilson 246d03b224fSChris Wilson out_fw: 247c1132367SAndi Shyti intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); 248092be382SChris Wilson intel_gt_pm_put(gt); 249092be382SChris Wilson return err; 250d03b224fSChris Wilson 251d03b224fSChris Wilson err_wedged: 252d03b224fSChris Wilson intel_gt_set_wedged(gt); 253d03b224fSChris Wilson goto out_fw; 254092be382SChris Wilson } 2559dfe3459SDaniele Ceraolo Spurio 256a70a9e99SChris Wilson static void wait_for_suspend(struct intel_gt *gt) 257c1132367SAndi Shyti { 258a70a9e99SChris Wilson if (!intel_gt_pm_is_awake(gt)) 259a70a9e99SChris Wilson return; 260a70a9e99SChris Wilson 26166101975SChris Wilson if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) { 262c1132367SAndi Shyti /* 263c1132367SAndi Shyti * Forcibly cancel outstanding work and leave 264c1132367SAndi Shyti * the gpu quiet. 265c1132367SAndi Shyti */ 266c1132367SAndi Shyti intel_gt_set_wedged(gt); 2670cdfdf6fSChris Wilson intel_gt_retire_requests(gt); 268c1132367SAndi Shyti } 269c1132367SAndi Shyti 270c1132367SAndi Shyti intel_gt_pm_wait_for_idle(gt); 271c1132367SAndi Shyti } 272c1132367SAndi Shyti 273a70a9e99SChris Wilson void intel_gt_suspend_prepare(struct intel_gt *gt) 274a70a9e99SChris Wilson { 275a70a9e99SChris Wilson user_forcewake(gt, true); 276a70a9e99SChris Wilson wait_for_suspend(gt); 277a70a9e99SChris Wilson 278a70a9e99SChris Wilson intel_uc_suspend(>->uc); 279a70a9e99SChris Wilson } 280a70a9e99SChris Wilson 281a70a9e99SChris Wilson static suspend_state_t pm_suspend_target(void) 282a70a9e99SChris Wilson { 283e435c608SChris Wilson #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP) 284a70a9e99SChris Wilson return pm_suspend_target_state; 285a70a9e99SChris Wilson #else 286a70a9e99SChris Wilson return PM_SUSPEND_TO_IDLE; 287a70a9e99SChris Wilson #endif 288a70a9e99SChris Wilson } 289a70a9e99SChris Wilson 290a70a9e99SChris Wilson void intel_gt_suspend_late(struct intel_gt *gt) 291c1132367SAndi Shyti { 292c1132367SAndi Shyti intel_wakeref_t wakeref; 293c1132367SAndi Shyti 294c1132367SAndi Shyti /* We expect to be idle already; but also want to be independent */ 295a70a9e99SChris Wilson wait_for_suspend(gt); 296c1132367SAndi Shyti 297e26b6d43SChris Wilson if (is_mock_gt(gt)) 298e26b6d43SChris Wilson return; 299e26b6d43SChris Wilson 300e26b6d43SChris Wilson GEM_BUG_ON(gt->awake); 301e26b6d43SChris Wilson 302a70a9e99SChris Wilson /* 303a70a9e99SChris Wilson * On disabling the device, we want to turn off HW access to memory 304a70a9e99SChris Wilson * that we no longer own. 305a70a9e99SChris Wilson * 306a70a9e99SChris Wilson * However, not all suspend-states disable the device. S0 (s2idle) 307a70a9e99SChris Wilson * is effectively runtime-suspend, the device is left powered on 308a70a9e99SChris Wilson * but needs to be put into a low power state. We need to keep 309a70a9e99SChris Wilson * powermanagement enabled, but we also retain system state and so 310a70a9e99SChris Wilson * it remains safe to keep on using our allocated memory. 311a70a9e99SChris Wilson */ 312a70a9e99SChris Wilson if (pm_suspend_target() == PM_SUSPEND_TO_IDLE) 313a70a9e99SChris Wilson return; 314fd6fe087SChris Wilson 3153e7abf81SAndi Shyti with_intel_runtime_pm(gt->uncore->rpm, wakeref) { 3163e7abf81SAndi Shyti intel_rps_disable(>->rps); 317c1132367SAndi Shyti intel_rc6_disable(>->rc6); 3183e7abf81SAndi Shyti intel_llc_disable(>->llc); 3193e7abf81SAndi Shyti } 320fd6fe087SChris Wilson 321d03b224fSChris Wilson gt_sanitize(gt, false); 322fd6fe087SChris Wilson 323639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "\n"); 324c1132367SAndi Shyti } 325c1132367SAndi Shyti 3269dfe3459SDaniele Ceraolo Spurio void intel_gt_runtime_suspend(struct intel_gt *gt) 3279dfe3459SDaniele Ceraolo Spurio { 3289dfe3459SDaniele Ceraolo Spurio intel_uc_runtime_suspend(>->uc); 329fd6fe087SChris Wilson 330639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "\n"); 3319dfe3459SDaniele Ceraolo Spurio } 3329dfe3459SDaniele Ceraolo Spurio 3339dfe3459SDaniele Ceraolo Spurio int intel_gt_runtime_resume(struct intel_gt *gt) 3349dfe3459SDaniele Ceraolo Spurio { 335639f2f24SVenkata Sandeep Dhanalakota GT_TRACE(gt, "\n"); 3369dfe3459SDaniele Ceraolo Spurio intel_gt_init_swizzling(gt); 337dec9cf9eSChris Wilson intel_ggtt_restore_fences(gt->ggtt); 3389dfe3459SDaniele Ceraolo Spurio 3399dfe3459SDaniele Ceraolo Spurio return intel_uc_runtime_resume(>->uc); 3409dfe3459SDaniele Ceraolo Spurio } 341c1132367SAndi Shyti 342c1132367SAndi Shyti #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 343c1132367SAndi Shyti #include "selftest_gt_pm.c" 344c1132367SAndi Shyti #endif 345