1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include <linux/string_helpers.h> 7 #include <linux/suspend.h> 8 9 #include "i915_drv.h" 10 #include "i915_params.h" 11 #include "intel_context.h" 12 #include "intel_engine_pm.h" 13 #include "intel_gt.h" 14 #include "intel_gt_clock_utils.h" 15 #include "intel_gt_pm.h" 16 #include "intel_gt_requests.h" 17 #include "intel_llc.h" 18 #include "intel_pm.h" 19 #include "intel_rc6.h" 20 #include "intel_rps.h" 21 #include "intel_wakeref.h" 22 #include "intel_pcode.h" 23 #include "pxp/intel_pxp_pm.h" 24 25 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2) 26 27 static void mtl_media_busy(struct intel_gt *gt) 28 { 29 /* Wa_14017073508: mtl */ 30 if (IS_MTL_GRAPHICS_STEP(gt->i915, P, STEP_A0, STEP_B0) && 31 gt->type == GT_MEDIA) 32 snb_pcode_write_p(gt->uncore, PCODE_MBOX_GT_STATE, 33 PCODE_MBOX_GT_STATE_MEDIA_BUSY, 34 PCODE_MBOX_GT_STATE_DOMAIN_MEDIA, 0); 35 } 36 37 static void mtl_media_idle(struct intel_gt *gt) 38 { 39 /* Wa_14017073508: mtl */ 40 if (IS_MTL_GRAPHICS_STEP(gt->i915, P, STEP_A0, STEP_B0) && 41 gt->type == GT_MEDIA) 42 snb_pcode_write_p(gt->uncore, PCODE_MBOX_GT_STATE, 43 PCODE_MBOX_GT_STATE_MEDIA_NOT_BUSY, 44 PCODE_MBOX_GT_STATE_DOMAIN_MEDIA, 0); 45 } 46 47 static void user_forcewake(struct intel_gt *gt, bool suspend) 48 { 49 int count = atomic_read(>->user_wakeref); 50 51 /* Inside suspend/resume so single threaded, no races to worry about. */ 52 if (likely(!count)) 53 return; 54 55 intel_gt_pm_get(gt); 56 if (suspend) { 57 GEM_BUG_ON(count > atomic_read(>->wakeref.count)); 58 atomic_sub(count, >->wakeref.count); 59 } else { 60 atomic_add(count, >->wakeref.count); 61 } 62 intel_gt_pm_put(gt); 63 } 64 65 static void runtime_begin(struct intel_gt *gt) 66 { 67 local_irq_disable(); 68 write_seqcount_begin(>->stats.lock); 69 gt->stats.start = ktime_get(); 70 gt->stats.active = true; 71 write_seqcount_end(>->stats.lock); 72 local_irq_enable(); 73 } 74 75 static void runtime_end(struct intel_gt *gt) 76 { 77 local_irq_disable(); 78 write_seqcount_begin(>->stats.lock); 79 gt->stats.active = false; 80 gt->stats.total = 81 ktime_add(gt->stats.total, 82 ktime_sub(ktime_get(), gt->stats.start)); 83 write_seqcount_end(>->stats.lock); 84 local_irq_enable(); 85 } 86 87 static int __gt_unpark(struct intel_wakeref *wf) 88 { 89 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); 90 struct drm_i915_private *i915 = gt->i915; 91 92 GT_TRACE(gt, "\n"); 93 94 /* Wa_14017073508: mtl */ 95 mtl_media_busy(gt); 96 97 /* 98 * It seems that the DMC likes to transition between the DC states a lot 99 * when there are no connected displays (no active power domains) during 100 * command submission. 101 * 102 * This activity has negative impact on the performance of the chip with 103 * huge latencies observed in the interrupt handler and elsewhere. 104 * 105 * Work around it by grabbing a GT IRQ power domain whilst there is any 106 * GT activity, preventing any DC state transitions. 107 */ 108 gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ); 109 GEM_BUG_ON(!gt->awake); 110 111 intel_rc6_unpark(>->rc6); 112 intel_rps_unpark(>->rps); 113 i915_pmu_gt_unparked(i915); 114 intel_guc_busyness_unpark(gt); 115 116 intel_gt_unpark_requests(gt); 117 runtime_begin(gt); 118 119 return 0; 120 } 121 122 static int __gt_park(struct intel_wakeref *wf) 123 { 124 struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref); 125 intel_wakeref_t wakeref = fetch_and_zero(>->awake); 126 struct drm_i915_private *i915 = gt->i915; 127 128 GT_TRACE(gt, "\n"); 129 130 runtime_end(gt); 131 intel_gt_park_requests(gt); 132 133 intel_guc_busyness_park(gt); 134 i915_vma_parked(gt); 135 i915_pmu_gt_parked(i915); 136 intel_rps_park(>->rps); 137 intel_rc6_park(>->rc6); 138 139 /* Everything switched off, flush any residual interrupt just in case */ 140 intel_synchronize_irq(i915); 141 142 /* Defer dropping the display power well for 100ms, it's slow! */ 143 GEM_BUG_ON(!wakeref); 144 intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref); 145 146 /* Wa_14017073508: mtl */ 147 mtl_media_idle(gt); 148 149 return 0; 150 } 151 152 static const struct intel_wakeref_ops wf_ops = { 153 .get = __gt_unpark, 154 .put = __gt_park, 155 }; 156 157 void intel_gt_pm_init_early(struct intel_gt *gt) 158 { 159 /* 160 * We access the runtime_pm structure via gt->i915 here rather than 161 * gt->uncore as we do elsewhere in the file because gt->uncore is not 162 * yet initialized for all tiles at this point in the driver startup. 163 * runtime_pm is per-device rather than per-tile, so this is still the 164 * correct structure. 165 */ 166 intel_wakeref_init(>->wakeref, >->i915->runtime_pm, &wf_ops); 167 seqcount_mutex_init(>->stats.lock, >->wakeref.mutex); 168 } 169 170 void intel_gt_pm_init(struct intel_gt *gt) 171 { 172 /* 173 * Enabling power-management should be "self-healing". If we cannot 174 * enable a feature, simply leave it disabled with a notice to the 175 * user. 176 */ 177 intel_rc6_init(>->rc6); 178 intel_rps_init(>->rps); 179 } 180 181 static bool reset_engines(struct intel_gt *gt) 182 { 183 if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display) 184 return false; 185 186 return __intel_gt_reset(gt, ALL_ENGINES) == 0; 187 } 188 189 static void gt_sanitize(struct intel_gt *gt, bool force) 190 { 191 struct intel_engine_cs *engine; 192 enum intel_engine_id id; 193 intel_wakeref_t wakeref; 194 195 GT_TRACE(gt, "force:%s", str_yes_no(force)); 196 197 /* Use a raw wakeref to avoid calling intel_display_power_get early */ 198 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 199 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); 200 201 intel_gt_check_clock_frequency(gt); 202 203 /* 204 * As we have just resumed the machine and woken the device up from 205 * deep PCI sleep (presumably D3_cold), assume the HW has been reset 206 * back to defaults, recovering from whatever wedged state we left it 207 * in and so worth trying to use the device once more. 208 */ 209 if (intel_gt_is_wedged(gt)) 210 intel_gt_unset_wedged(gt); 211 212 /* For GuC mode, ensure submission is disabled before stopping ring */ 213 intel_uc_reset_prepare(>->uc); 214 215 for_each_engine(engine, gt, id) { 216 if (engine->reset.prepare) 217 engine->reset.prepare(engine); 218 219 if (engine->sanitize) 220 engine->sanitize(engine); 221 } 222 223 if (reset_engines(gt) || force) { 224 for_each_engine(engine, gt, id) 225 __intel_engine_reset(engine, false); 226 } 227 228 intel_uc_reset(>->uc, false); 229 230 for_each_engine(engine, gt, id) 231 if (engine->reset.finish) 232 engine->reset.finish(engine); 233 234 intel_rps_sanitize(>->rps); 235 236 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); 237 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 238 } 239 240 void intel_gt_pm_fini(struct intel_gt *gt) 241 { 242 intel_rc6_fini(>->rc6); 243 } 244 245 int intel_gt_resume(struct intel_gt *gt) 246 { 247 struct intel_engine_cs *engine; 248 enum intel_engine_id id; 249 int err; 250 251 err = intel_gt_has_unrecoverable_error(gt); 252 if (err) 253 return err; 254 255 GT_TRACE(gt, "\n"); 256 257 /* 258 * After resume, we may need to poke into the pinned kernel 259 * contexts to paper over any damage caused by the sudden suspend. 260 * Only the kernel contexts should remain pinned over suspend, 261 * allowing us to fixup the user contexts on their first pin. 262 */ 263 gt_sanitize(gt, true); 264 265 intel_gt_pm_get(gt); 266 267 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); 268 intel_rc6_sanitize(>->rc6); 269 if (intel_gt_is_wedged(gt)) { 270 err = -EIO; 271 goto out_fw; 272 } 273 274 /* Only when the HW is re-initialised, can we replay the requests */ 275 err = intel_gt_init_hw(gt); 276 if (err) { 277 i915_probe_error(gt->i915, 278 "Failed to initialize GPU, declaring it wedged!\n"); 279 goto err_wedged; 280 } 281 282 intel_uc_reset_finish(>->uc); 283 284 intel_rps_enable(>->rps); 285 intel_llc_enable(>->llc); 286 287 for_each_engine(engine, gt, id) { 288 intel_engine_pm_get(engine); 289 290 engine->serial++; /* kernel context lost */ 291 err = intel_engine_resume(engine); 292 293 intel_engine_pm_put(engine); 294 if (err) { 295 drm_err(>->i915->drm, 296 "Failed to restart %s (%d)\n", 297 engine->name, err); 298 goto err_wedged; 299 } 300 } 301 302 intel_rc6_enable(>->rc6); 303 304 intel_uc_resume(>->uc); 305 306 intel_pxp_resume(>->pxp); 307 308 user_forcewake(gt, false); 309 310 out_fw: 311 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); 312 intel_gt_pm_put(gt); 313 return err; 314 315 err_wedged: 316 intel_gt_set_wedged(gt); 317 goto out_fw; 318 } 319 320 static void wait_for_suspend(struct intel_gt *gt) 321 { 322 if (!intel_gt_pm_is_awake(gt)) 323 return; 324 325 if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) { 326 /* 327 * Forcibly cancel outstanding work and leave 328 * the gpu quiet. 329 */ 330 intel_gt_set_wedged(gt); 331 intel_gt_retire_requests(gt); 332 } 333 334 intel_gt_pm_wait_for_idle(gt); 335 } 336 337 void intel_gt_suspend_prepare(struct intel_gt *gt) 338 { 339 user_forcewake(gt, true); 340 wait_for_suspend(gt); 341 342 intel_pxp_suspend_prepare(>->pxp); 343 } 344 345 static suspend_state_t pm_suspend_target(void) 346 { 347 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP) 348 return pm_suspend_target_state; 349 #else 350 return PM_SUSPEND_TO_IDLE; 351 #endif 352 } 353 354 void intel_gt_suspend_late(struct intel_gt *gt) 355 { 356 intel_wakeref_t wakeref; 357 358 /* We expect to be idle already; but also want to be independent */ 359 wait_for_suspend(gt); 360 361 if (is_mock_gt(gt)) 362 return; 363 364 GEM_BUG_ON(gt->awake); 365 366 intel_uc_suspend(>->uc); 367 intel_pxp_suspend(>->pxp); 368 369 /* 370 * On disabling the device, we want to turn off HW access to memory 371 * that we no longer own. 372 * 373 * However, not all suspend-states disable the device. S0 (s2idle) 374 * is effectively runtime-suspend, the device is left powered on 375 * but needs to be put into a low power state. We need to keep 376 * powermanagement enabled, but we also retain system state and so 377 * it remains safe to keep on using our allocated memory. 378 */ 379 if (pm_suspend_target() == PM_SUSPEND_TO_IDLE) 380 return; 381 382 with_intel_runtime_pm(gt->uncore->rpm, wakeref) { 383 intel_rps_disable(>->rps); 384 intel_rc6_disable(>->rc6); 385 intel_llc_disable(>->llc); 386 } 387 388 gt_sanitize(gt, false); 389 390 GT_TRACE(gt, "\n"); 391 } 392 393 void intel_gt_runtime_suspend(struct intel_gt *gt) 394 { 395 intel_pxp_runtime_suspend(>->pxp); 396 intel_uc_runtime_suspend(>->uc); 397 398 GT_TRACE(gt, "\n"); 399 } 400 401 int intel_gt_runtime_resume(struct intel_gt *gt) 402 { 403 int ret; 404 405 GT_TRACE(gt, "\n"); 406 intel_gt_init_swizzling(gt); 407 intel_ggtt_restore_fences(gt->ggtt); 408 409 ret = intel_uc_runtime_resume(>->uc); 410 if (ret) 411 return ret; 412 413 intel_pxp_runtime_resume(>->pxp); 414 415 return 0; 416 } 417 418 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt) 419 { 420 ktime_t total = gt->stats.total; 421 422 if (gt->stats.active) 423 total = ktime_add(total, 424 ktime_sub(ktime_get(), gt->stats.start)); 425 426 return total; 427 } 428 429 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt) 430 { 431 unsigned int seq; 432 ktime_t total; 433 434 do { 435 seq = read_seqcount_begin(>->stats.lock); 436 total = __intel_gt_get_awake_time(gt); 437 } while (read_seqcount_retry(>->stats.lock, seq)); 438 439 return total; 440 } 441 442 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 443 #include "selftest_gt_pm.c" 444 #endif 445