xref: /openbmc/linux/drivers/gpu/drm/i915/gt/intel_gt_pm.c (revision e7f127b2)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/suspend.h>
7 
8 #include "i915_drv.h"
9 #include "i915_params.h"
10 #include "intel_context.h"
11 #include "intel_engine_pm.h"
12 #include "intel_gt.h"
13 #include "intel_gt_clock_utils.h"
14 #include "intel_gt_pm.h"
15 #include "intel_gt_requests.h"
16 #include "intel_llc.h"
17 #include "intel_pm.h"
18 #include "intel_rc6.h"
19 #include "intel_rps.h"
20 #include "intel_wakeref.h"
21 #include "pxp/intel_pxp_pm.h"
22 
23 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
24 
25 static void user_forcewake(struct intel_gt *gt, bool suspend)
26 {
27 	int count = atomic_read(&gt->user_wakeref);
28 
29 	/* Inside suspend/resume so single threaded, no races to worry about. */
30 	if (likely(!count))
31 		return;
32 
33 	intel_gt_pm_get(gt);
34 	if (suspend) {
35 		GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
36 		atomic_sub(count, &gt->wakeref.count);
37 	} else {
38 		atomic_add(count, &gt->wakeref.count);
39 	}
40 	intel_gt_pm_put(gt);
41 }
42 
43 static void runtime_begin(struct intel_gt *gt)
44 {
45 	local_irq_disable();
46 	write_seqcount_begin(&gt->stats.lock);
47 	gt->stats.start = ktime_get();
48 	gt->stats.active = true;
49 	write_seqcount_end(&gt->stats.lock);
50 	local_irq_enable();
51 }
52 
53 static void runtime_end(struct intel_gt *gt)
54 {
55 	local_irq_disable();
56 	write_seqcount_begin(&gt->stats.lock);
57 	gt->stats.active = false;
58 	gt->stats.total =
59 		ktime_add(gt->stats.total,
60 			  ktime_sub(ktime_get(), gt->stats.start));
61 	write_seqcount_end(&gt->stats.lock);
62 	local_irq_enable();
63 }
64 
65 static int __gt_unpark(struct intel_wakeref *wf)
66 {
67 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
68 	struct drm_i915_private *i915 = gt->i915;
69 
70 	GT_TRACE(gt, "\n");
71 
72 	/*
73 	 * It seems that the DMC likes to transition between the DC states a lot
74 	 * when there are no connected displays (no active power domains) during
75 	 * command submission.
76 	 *
77 	 * This activity has negative impact on the performance of the chip with
78 	 * huge latencies observed in the interrupt handler and elsewhere.
79 	 *
80 	 * Work around it by grabbing a GT IRQ power domain whilst there is any
81 	 * GT activity, preventing any DC state transitions.
82 	 */
83 	gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
84 	GEM_BUG_ON(!gt->awake);
85 
86 	intel_rc6_unpark(&gt->rc6);
87 	intel_rps_unpark(&gt->rps);
88 	i915_pmu_gt_unparked(i915);
89 	intel_guc_busyness_unpark(gt);
90 
91 	intel_gt_unpark_requests(gt);
92 	runtime_begin(gt);
93 
94 	return 0;
95 }
96 
97 static int __gt_park(struct intel_wakeref *wf)
98 {
99 	struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
100 	intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
101 	struct drm_i915_private *i915 = gt->i915;
102 
103 	GT_TRACE(gt, "\n");
104 
105 	runtime_end(gt);
106 	intel_gt_park_requests(gt);
107 
108 	intel_guc_busyness_park(gt);
109 	i915_vma_parked(gt);
110 	i915_pmu_gt_parked(i915);
111 	intel_rps_park(&gt->rps);
112 	intel_rc6_park(&gt->rc6);
113 
114 	/* Everything switched off, flush any residual interrupt just in case */
115 	intel_synchronize_irq(i915);
116 
117 	/* Defer dropping the display power well for 100ms, it's slow! */
118 	GEM_BUG_ON(!wakeref);
119 	intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
120 
121 	return 0;
122 }
123 
124 static const struct intel_wakeref_ops wf_ops = {
125 	.get = __gt_unpark,
126 	.put = __gt_park,
127 };
128 
129 void intel_gt_pm_init_early(struct intel_gt *gt)
130 {
131 	intel_wakeref_init(&gt->wakeref, gt->uncore->rpm, &wf_ops);
132 	seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
133 }
134 
135 void intel_gt_pm_init(struct intel_gt *gt)
136 {
137 	/*
138 	 * Enabling power-management should be "self-healing". If we cannot
139 	 * enable a feature, simply leave it disabled with a notice to the
140 	 * user.
141 	 */
142 	intel_rc6_init(&gt->rc6);
143 	intel_rps_init(&gt->rps);
144 }
145 
146 static bool reset_engines(struct intel_gt *gt)
147 {
148 	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
149 		return false;
150 
151 	return __intel_gt_reset(gt, ALL_ENGINES) == 0;
152 }
153 
154 static void gt_sanitize(struct intel_gt *gt, bool force)
155 {
156 	struct intel_engine_cs *engine;
157 	enum intel_engine_id id;
158 	intel_wakeref_t wakeref;
159 
160 	GT_TRACE(gt, "force:%s", yesno(force));
161 
162 	/* Use a raw wakeref to avoid calling intel_display_power_get early */
163 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
164 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
165 
166 	intel_gt_check_clock_frequency(gt);
167 
168 	/*
169 	 * As we have just resumed the machine and woken the device up from
170 	 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
171 	 * back to defaults, recovering from whatever wedged state we left it
172 	 * in and so worth trying to use the device once more.
173 	 */
174 	if (intel_gt_is_wedged(gt))
175 		intel_gt_unset_wedged(gt);
176 
177 	for_each_engine(engine, gt, id)
178 		if (engine->reset.prepare)
179 			engine->reset.prepare(engine);
180 
181 	intel_uc_reset_prepare(&gt->uc);
182 
183 	for_each_engine(engine, gt, id)
184 		if (engine->sanitize)
185 			engine->sanitize(engine);
186 
187 	if (reset_engines(gt) || force) {
188 		for_each_engine(engine, gt, id)
189 			__intel_engine_reset(engine, false);
190 	}
191 
192 	intel_uc_reset(&gt->uc, false);
193 
194 	for_each_engine(engine, gt, id)
195 		if (engine->reset.finish)
196 			engine->reset.finish(engine);
197 
198 	intel_rps_sanitize(&gt->rps);
199 
200 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
201 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
202 }
203 
204 void intel_gt_pm_fini(struct intel_gt *gt)
205 {
206 	intel_rc6_fini(&gt->rc6);
207 }
208 
209 int intel_gt_resume(struct intel_gt *gt)
210 {
211 	struct intel_engine_cs *engine;
212 	enum intel_engine_id id;
213 	int err;
214 
215 	err = intel_gt_has_unrecoverable_error(gt);
216 	if (err)
217 		return err;
218 
219 	GT_TRACE(gt, "\n");
220 
221 	/*
222 	 * After resume, we may need to poke into the pinned kernel
223 	 * contexts to paper over any damage caused by the sudden suspend.
224 	 * Only the kernel contexts should remain pinned over suspend,
225 	 * allowing us to fixup the user contexts on their first pin.
226 	 */
227 	gt_sanitize(gt, true);
228 
229 	intel_gt_pm_get(gt);
230 
231 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
232 	intel_rc6_sanitize(&gt->rc6);
233 	if (intel_gt_is_wedged(gt)) {
234 		err = -EIO;
235 		goto out_fw;
236 	}
237 
238 	/* Only when the HW is re-initialised, can we replay the requests */
239 	err = intel_gt_init_hw(gt);
240 	if (err) {
241 		i915_probe_error(gt->i915,
242 				 "Failed to initialize GPU, declaring it wedged!\n");
243 		goto err_wedged;
244 	}
245 
246 	intel_uc_reset_finish(&gt->uc);
247 
248 	intel_rps_enable(&gt->rps);
249 	intel_llc_enable(&gt->llc);
250 
251 	for_each_engine(engine, gt, id) {
252 		intel_engine_pm_get(engine);
253 
254 		engine->serial++; /* kernel context lost */
255 		err = intel_engine_resume(engine);
256 
257 		intel_engine_pm_put(engine);
258 		if (err) {
259 			drm_err(&gt->i915->drm,
260 				"Failed to restart %s (%d)\n",
261 				engine->name, err);
262 			goto err_wedged;
263 		}
264 	}
265 
266 	intel_rc6_enable(&gt->rc6);
267 
268 	intel_uc_resume(&gt->uc);
269 
270 	intel_pxp_resume(&gt->pxp);
271 
272 	user_forcewake(gt, false);
273 
274 out_fw:
275 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
276 	intel_gt_pm_put(gt);
277 	return err;
278 
279 err_wedged:
280 	intel_gt_set_wedged(gt);
281 	goto out_fw;
282 }
283 
284 static void wait_for_suspend(struct intel_gt *gt)
285 {
286 	if (!intel_gt_pm_is_awake(gt))
287 		return;
288 
289 	if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
290 		/*
291 		 * Forcibly cancel outstanding work and leave
292 		 * the gpu quiet.
293 		 */
294 		intel_gt_set_wedged(gt);
295 		intel_gt_retire_requests(gt);
296 	}
297 
298 	intel_gt_pm_wait_for_idle(gt);
299 }
300 
301 void intel_gt_suspend_prepare(struct intel_gt *gt)
302 {
303 	user_forcewake(gt, true);
304 	wait_for_suspend(gt);
305 
306 	intel_pxp_suspend_prepare(&gt->pxp);
307 }
308 
309 static suspend_state_t pm_suspend_target(void)
310 {
311 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
312 	return pm_suspend_target_state;
313 #else
314 	return PM_SUSPEND_TO_IDLE;
315 #endif
316 }
317 
318 void intel_gt_suspend_late(struct intel_gt *gt)
319 {
320 	intel_wakeref_t wakeref;
321 
322 	/* We expect to be idle already; but also want to be independent */
323 	wait_for_suspend(gt);
324 
325 	if (is_mock_gt(gt))
326 		return;
327 
328 	GEM_BUG_ON(gt->awake);
329 
330 	intel_uc_suspend(&gt->uc);
331 	intel_pxp_suspend(&gt->pxp);
332 
333 	/*
334 	 * On disabling the device, we want to turn off HW access to memory
335 	 * that we no longer own.
336 	 *
337 	 * However, not all suspend-states disable the device. S0 (s2idle)
338 	 * is effectively runtime-suspend, the device is left powered on
339 	 * but needs to be put into a low power state. We need to keep
340 	 * powermanagement enabled, but we also retain system state and so
341 	 * it remains safe to keep on using our allocated memory.
342 	 */
343 	if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
344 		return;
345 
346 	with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
347 		intel_rps_disable(&gt->rps);
348 		intel_rc6_disable(&gt->rc6);
349 		intel_llc_disable(&gt->llc);
350 	}
351 
352 	gt_sanitize(gt, false);
353 
354 	GT_TRACE(gt, "\n");
355 }
356 
357 void intel_gt_runtime_suspend(struct intel_gt *gt)
358 {
359 	intel_pxp_runtime_suspend(&gt->pxp);
360 	intel_uc_runtime_suspend(&gt->uc);
361 
362 	GT_TRACE(gt, "\n");
363 }
364 
365 int intel_gt_runtime_resume(struct intel_gt *gt)
366 {
367 	int ret;
368 
369 	GT_TRACE(gt, "\n");
370 	intel_gt_init_swizzling(gt);
371 	intel_ggtt_restore_fences(gt->ggtt);
372 
373 	ret = intel_uc_runtime_resume(&gt->uc);
374 	if (ret)
375 		return ret;
376 
377 	intel_pxp_runtime_resume(&gt->pxp);
378 
379 	return 0;
380 }
381 
382 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
383 {
384 	ktime_t total = gt->stats.total;
385 
386 	if (gt->stats.active)
387 		total = ktime_add(total,
388 				  ktime_sub(ktime_get(), gt->stats.start));
389 
390 	return total;
391 }
392 
393 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
394 {
395 	unsigned int seq;
396 	ktime_t total;
397 
398 	do {
399 		seq = read_seqcount_begin(&gt->stats.lock);
400 		total = __intel_gt_get_awake_time(gt);
401 	} while (read_seqcount_retry(&gt->stats.lock, seq));
402 
403 	return total;
404 }
405 
406 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
407 #include "selftest_gt_pm.c"
408 #endif
409