27 static void user_forcewake(struct intel_gt *gt, bool suspend)  in user_forcewake()  argument
29 	int count = atomic_read(&gt->user_wakeref);  in user_forcewake()
35 	intel_gt_pm_get(gt);  in user_forcewake()
37 		GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));  in user_forcewake()
38 		atomic_sub(count, &gt->wakeref.count);  in user_forcewake()
40 		atomic_add(count, &gt->wakeref.count);  in user_forcewake()
42 	intel_gt_pm_put(gt);  in user_forcewake()
45 static void runtime_begin(struct intel_gt *gt)  in runtime_begin()  argument
48 	write_seqcount_begin(&gt->stats.lock);  in runtime_begin()
49 	gt->stats.start = ktime_get();  in runtime_begin()
50 	gt->stats.active = true;  in runtime_begin()
51 	write_seqcount_end(&gt->stats.lock);  in runtime_begin()
55 static void runtime_end(struct intel_gt *gt)  in runtime_end()  argument
58 	write_seqcount_begin(&gt->stats.lock);  in runtime_end()
59 	gt->stats.active = false;  in runtime_end()
60 	gt->stats.total =  in runtime_end()
61 		ktime_add(gt->stats.total,  in runtime_end()
62 			  ktime_sub(ktime_get(), gt->stats.start));  in runtime_end()
63 	write_seqcount_end(&gt->stats.lock);  in runtime_end()
69 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);  in __gt_unpark()  local
70 	struct drm_i915_private *i915 = gt->i915;  in __gt_unpark()
72 	GT_TRACE(gt, "\n");  in __gt_unpark()
85 	gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);  in __gt_unpark()
86 	GEM_BUG_ON(!gt->awake);  in __gt_unpark()
88 	intel_rc6_unpark(&gt->rc6);  in __gt_unpark()
89 	intel_rps_unpark(&gt->rps);  in __gt_unpark()
90 	i915_pmu_gt_unparked(gt);  in __gt_unpark()
91 	intel_guc_busyness_unpark(gt);  in __gt_unpark()
93 	intel_gt_unpark_requests(gt);  in __gt_unpark()
94 	runtime_begin(gt);  in __gt_unpark()
101 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);  in __gt_park()  local
102 	intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);  in __gt_park()
103 	struct drm_i915_private *i915 = gt->i915;  in __gt_park()
105 	GT_TRACE(gt, "\n");  in __gt_park()
107 	runtime_end(gt);  in __gt_park()
108 	intel_gt_park_requests(gt);  in __gt_park()
110 	intel_guc_busyness_park(gt);  in __gt_park()
111 	i915_vma_parked(gt);  in __gt_park()
112 	i915_pmu_gt_parked(gt);  in __gt_park()
113 	intel_rps_park(&gt->rps);  in __gt_park()
114 	intel_rc6_park(&gt->rc6);  in __gt_park()
131 void intel_gt_pm_init_early(struct intel_gt *gt)  in intel_gt_pm_init_early()  argument
140 	intel_wakeref_init(&gt->wakeref, gt->i915, &wf_ops);  in intel_gt_pm_init_early()
141 	seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);  in intel_gt_pm_init_early()
144 void intel_gt_pm_init(struct intel_gt *gt)  in intel_gt_pm_init()  argument
151 	intel_rc6_init(&gt->rc6);  in intel_gt_pm_init()
152 	intel_rps_init(&gt->rps);  in intel_gt_pm_init()
155 static bool reset_engines(struct intel_gt *gt)  in reset_engines()  argument
157 	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)  in reset_engines()
160 	return __intel_gt_reset(gt, ALL_ENGINES) == 0;  in reset_engines()
163 static void gt_sanitize(struct intel_gt *gt, bool force)  in gt_sanitize()  argument
169 	GT_TRACE(gt, "force:%s", str_yes_no(force));  in gt_sanitize()
172 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);  in gt_sanitize()
173 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);  in gt_sanitize()
175 	intel_gt_check_clock_frequency(gt);  in gt_sanitize()
183 	if (intel_gt_is_wedged(gt))  in gt_sanitize()
184 		intel_gt_unset_wedged(gt);  in gt_sanitize()
187 	intel_uc_reset_prepare(&gt->uc);  in gt_sanitize()
189 	for_each_engine(engine, gt, id) {  in gt_sanitize()
197 	if (reset_engines(gt) || force) {  in gt_sanitize()
198 		for_each_engine(engine, gt, id)  in gt_sanitize()
202 	intel_uc_reset(&gt->uc, false);  in gt_sanitize()
204 	for_each_engine(engine, gt, id)  in gt_sanitize()
208 	intel_rps_sanitize(&gt->rps);  in gt_sanitize()
210 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);  in gt_sanitize()
211 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);  in gt_sanitize()
214 void intel_gt_pm_fini(struct intel_gt *gt)  in intel_gt_pm_fini()  argument
216 	intel_rc6_fini(&gt->rc6);  in intel_gt_pm_fini()
219 int intel_gt_resume(struct intel_gt *gt)  in intel_gt_resume()  argument
225 	err = intel_gt_has_unrecoverable_error(gt);  in intel_gt_resume()
229 	GT_TRACE(gt, "\n");  in intel_gt_resume()
237 	gt_sanitize(gt, true);  in intel_gt_resume()
239 	intel_gt_pm_get(gt);  in intel_gt_resume()
241 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);  in intel_gt_resume()
242 	intel_rc6_sanitize(&gt->rc6);  in intel_gt_resume()
243 	if (intel_gt_is_wedged(gt)) {  in intel_gt_resume()
249 	err = intel_gt_init_hw(gt);  in intel_gt_resume()
251 		gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n");  in intel_gt_resume()
255 	intel_uc_reset_finish(&gt->uc);  in intel_gt_resume()
257 	intel_rps_enable(&gt->rps);  in intel_gt_resume()
258 	intel_llc_enable(&gt->llc);  in intel_gt_resume()
260 	for_each_engine(engine, gt, id) {  in intel_gt_resume()
268 			gt_err(gt, "Failed to restart %s (%d)\n",  in intel_gt_resume()
274 	intel_rc6_enable(&gt->rc6);  in intel_gt_resume()
276 	intel_uc_resume(&gt->uc);  in intel_gt_resume()
278 	user_forcewake(gt, false);  in intel_gt_resume()
281 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);  in intel_gt_resume()
282 	intel_gt_pm_put(gt);  in intel_gt_resume()
286 	intel_gt_set_wedged(gt);  in intel_gt_resume()
290 static void wait_for_suspend(struct intel_gt *gt)  in wait_for_suspend()  argument
292 	if (!intel_gt_pm_is_awake(gt))  in wait_for_suspend()
295 	if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {  in wait_for_suspend()
300 		intel_gt_set_wedged(gt);  in wait_for_suspend()
301 		intel_gt_retire_requests(gt);  in wait_for_suspend()
304 	intel_gt_pm_wait_for_idle(gt);  in wait_for_suspend()
307 void intel_gt_suspend_prepare(struct intel_gt *gt)  in intel_gt_suspend_prepare()  argument
309 	user_forcewake(gt, true);  in intel_gt_suspend_prepare()
310 	wait_for_suspend(gt);  in intel_gt_suspend_prepare()
322 void intel_gt_suspend_late(struct intel_gt *gt)  in intel_gt_suspend_late()  argument
327 	wait_for_suspend(gt);  in intel_gt_suspend_late()
329 	if (is_mock_gt(gt))  in intel_gt_suspend_late()
332 	GEM_BUG_ON(gt->awake);  in intel_gt_suspend_late()
334 	intel_uc_suspend(&gt->uc);  in intel_gt_suspend_late()
349 	with_intel_runtime_pm(gt->uncore->rpm, wakeref) {  in intel_gt_suspend_late()
350 		intel_rps_disable(&gt->rps);  in intel_gt_suspend_late()
351 		intel_rc6_disable(&gt->rc6);  in intel_gt_suspend_late()
352 		intel_llc_disable(&gt->llc);  in intel_gt_suspend_late()
355 	gt_sanitize(gt, false);  in intel_gt_suspend_late()
357 	GT_TRACE(gt, "\n");  in intel_gt_suspend_late()
360 void intel_gt_runtime_suspend(struct intel_gt *gt)  in intel_gt_runtime_suspend()  argument
362 	intel_uc_runtime_suspend(&gt->uc);  in intel_gt_runtime_suspend()
364 	GT_TRACE(gt, "\n");  in intel_gt_runtime_suspend()
367 int intel_gt_runtime_resume(struct intel_gt *gt)  in intel_gt_runtime_resume()  argument
371 	GT_TRACE(gt, "\n");  in intel_gt_runtime_resume()
372 	intel_gt_init_swizzling(gt);  in intel_gt_runtime_resume()
373 	intel_ggtt_restore_fences(gt->ggtt);  in intel_gt_runtime_resume()
375 	ret = intel_uc_runtime_resume(&gt->uc);  in intel_gt_runtime_resume()
382 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)  in __intel_gt_get_awake_time()  argument
384 	ktime_t total = gt->stats.total;  in __intel_gt_get_awake_time()
386 	if (gt->stats.active)  in __intel_gt_get_awake_time()
388 				  ktime_sub(ktime_get(), gt->stats.start));  in __intel_gt_get_awake_time()
393 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)  in intel_gt_get_awake_time()  argument
399 		seq = read_seqcount_begin(&gt->stats.lock);  in intel_gt_get_awake_time()
400 		total = __intel_gt_get_awake_time(gt);  in intel_gt_get_awake_time()
401 	} while (read_seqcount_retry(&gt->stats.lock, seq));  in intel_gt_get_awake_time()