1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2019 Intel Corporation
5  */
6 
7 #include "gem/i915_gem_pm.h"
8 #include "gem/i915_gem_ttm_pm.h"
9 #include "gt/intel_gt.h"
10 #include "gt/intel_gt_pm.h"
11 #include "gt/intel_gt_requests.h"
12 
13 #include "i915_driver.h"
14 #include "i915_drv.h"
15 
16 #if defined(CONFIG_X86)
17 #include <asm/smp.h>
18 #else
19 #define wbinvd_on_all_cpus() \
20 	pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__)
21 #endif
22 
i915_gem_suspend(struct drm_i915_private * i915)23 void i915_gem_suspend(struct drm_i915_private *i915)
24 {
25 	struct intel_gt *gt;
26 	unsigned int i;
27 
28 	GEM_TRACE("%s\n", dev_name(i915->drm.dev));
29 
30 	intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref, 0);
31 	flush_workqueue(i915->wq);
32 
33 	/*
34 	 * We have to flush all the executing contexts to main memory so
35 	 * that they can saved in the hibernation image. To ensure the last
36 	 * context image is coherent, we have to switch away from it. That
37 	 * leaves the i915->kernel_context still active when
38 	 * we actually suspend, and its image in memory may not match the GPU
39 	 * state. Fortunately, the kernel_context is disposable and we do
40 	 * not rely on its state.
41 	 */
42 	for_each_gt(gt, i915, i)
43 		intel_gt_suspend_prepare(gt);
44 
45 	i915_gem_drain_freed_objects(i915);
46 }
47 
lmem_restore(struct drm_i915_private * i915,u32 flags)48 static int lmem_restore(struct drm_i915_private *i915, u32 flags)
49 {
50 	struct intel_memory_region *mr;
51 	int ret = 0, id;
52 
53 	for_each_memory_region(mr, i915, id) {
54 		if (mr->type == INTEL_MEMORY_LOCAL) {
55 			ret = i915_ttm_restore_region(mr, flags);
56 			if (ret)
57 				break;
58 		}
59 	}
60 
61 	return ret;
62 }
63 
lmem_suspend(struct drm_i915_private * i915,u32 flags)64 static int lmem_suspend(struct drm_i915_private *i915, u32 flags)
65 {
66 	struct intel_memory_region *mr;
67 	int ret = 0, id;
68 
69 	for_each_memory_region(mr, i915, id) {
70 		if (mr->type == INTEL_MEMORY_LOCAL) {
71 			ret = i915_ttm_backup_region(mr, flags);
72 			if (ret)
73 				break;
74 		}
75 	}
76 
77 	return ret;
78 }
79 
lmem_recover(struct drm_i915_private * i915)80 static void lmem_recover(struct drm_i915_private *i915)
81 {
82 	struct intel_memory_region *mr;
83 	int id;
84 
85 	for_each_memory_region(mr, i915, id)
86 		if (mr->type == INTEL_MEMORY_LOCAL)
87 			i915_ttm_recover_region(mr);
88 }
89 
i915_gem_backup_suspend(struct drm_i915_private * i915)90 int i915_gem_backup_suspend(struct drm_i915_private *i915)
91 {
92 	int ret;
93 
94 	/* Opportunistically try to evict unpinned objects */
95 	ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU);
96 	if (ret)
97 		goto out_recover;
98 
99 	i915_gem_suspend(i915);
100 
101 	/*
102 	 * More objects may have become unpinned as requests were
103 	 * retired. Now try to evict again. The gt may be wedged here
104 	 * in which case we automatically fall back to memcpy.
105 	 * We allow also backing up pinned objects that have not been
106 	 * marked for early recover, and that may contain, for example,
107 	 * page-tables for the migrate context.
108 	 */
109 	ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU |
110 			   I915_TTM_BACKUP_PINNED);
111 	if (ret)
112 		goto out_recover;
113 
114 	/*
115 	 * Remaining objects are backed up using memcpy once we've stopped
116 	 * using the migrate context.
117 	 */
118 	ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED);
119 	if (ret)
120 		goto out_recover;
121 
122 	return 0;
123 
124 out_recover:
125 	lmem_recover(i915);
126 
127 	return ret;
128 }
129 
i915_gem_suspend_late(struct drm_i915_private * i915)130 void i915_gem_suspend_late(struct drm_i915_private *i915)
131 {
132 	struct drm_i915_gem_object *obj;
133 	struct list_head *phases[] = {
134 		&i915->mm.shrink_list,
135 		&i915->mm.purge_list,
136 		NULL
137 	}, **phase;
138 	struct intel_gt *gt;
139 	unsigned long flags;
140 	unsigned int i;
141 	bool flush = false;
142 
143 	/*
144 	 * Neither the BIOS, ourselves or any other kernel
145 	 * expects the system to be in execlists mode on startup,
146 	 * so we need to reset the GPU back to legacy mode. And the only
147 	 * known way to disable logical contexts is through a GPU reset.
148 	 *
149 	 * So in order to leave the system in a known default configuration,
150 	 * always reset the GPU upon unload and suspend. Afterwards we then
151 	 * clean up the GEM state tracking, flushing off the requests and
152 	 * leaving the system in a known idle state.
153 	 *
154 	 * Note that is of the upmost importance that the GPU is idle and
155 	 * all stray writes are flushed *before* we dismantle the backing
156 	 * storage for the pinned objects.
157 	 *
158 	 * However, since we are uncertain that resetting the GPU on older
159 	 * machines is a good idea, we don't - just in case it leaves the
160 	 * machine in an unusable condition.
161 	 */
162 
163 	for_each_gt(gt, i915, i)
164 		intel_gt_suspend_late(gt);
165 
166 	spin_lock_irqsave(&i915->mm.obj_lock, flags);
167 	for (phase = phases; *phase; phase++) {
168 		list_for_each_entry(obj, *phase, mm.link) {
169 			if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
170 				flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0;
171 			__start_cpu_write(obj); /* presume auto-hibernate */
172 		}
173 	}
174 	spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
175 	if (flush)
176 		wbinvd_on_all_cpus();
177 }
178 
i915_gem_freeze(struct drm_i915_private * i915)179 int i915_gem_freeze(struct drm_i915_private *i915)
180 {
181 	/* Discard all purgeable objects, let userspace recover those as
182 	 * required after resuming.
183 	 */
184 	i915_gem_shrink_all(i915);
185 
186 	return 0;
187 }
188 
i915_gem_freeze_late(struct drm_i915_private * i915)189 int i915_gem_freeze_late(struct drm_i915_private *i915)
190 {
191 	struct drm_i915_gem_object *obj;
192 	intel_wakeref_t wakeref;
193 
194 	/*
195 	 * Called just before we write the hibernation image.
196 	 *
197 	 * We need to update the domain tracking to reflect that the CPU
198 	 * will be accessing all the pages to create and restore from the
199 	 * hibernation, and so upon restoration those pages will be in the
200 	 * CPU domain.
201 	 *
202 	 * To make sure the hibernation image contains the latest state,
203 	 * we update that state just before writing out the image.
204 	 *
205 	 * To try and reduce the hibernation image, we manually shrink
206 	 * the objects as well, see i915_gem_freeze()
207 	 */
208 
209 	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
210 		i915_gem_shrink(NULL, i915, -1UL, NULL, ~0);
211 	i915_gem_drain_freed_objects(i915);
212 
213 	wbinvd_on_all_cpus();
214 	list_for_each_entry(obj, &i915->mm.shrink_list, mm.link)
215 		__start_cpu_write(obj);
216 
217 	return 0;
218 }
219 
i915_gem_resume(struct drm_i915_private * i915)220 void i915_gem_resume(struct drm_i915_private *i915)
221 {
222 	struct intel_gt *gt;
223 	int ret, i, j;
224 
225 	GEM_TRACE("%s\n", dev_name(i915->drm.dev));
226 
227 	ret = lmem_restore(i915, 0);
228 	GEM_WARN_ON(ret);
229 
230 	/*
231 	 * As we didn't flush the kernel context before suspend, we cannot
232 	 * guarantee that the context image is complete. So let's just reset
233 	 * it and start again.
234 	 */
235 	for_each_gt(gt, i915, i)
236 		if (intel_gt_resume(gt))
237 			goto err_wedged;
238 
239 	ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU);
240 	GEM_WARN_ON(ret);
241 
242 	return;
243 
244 err_wedged:
245 	for_each_gt(gt, i915, j) {
246 		if (!intel_gt_is_wedged(gt)) {
247 			dev_err(i915->drm.dev,
248 				"Failed to re-initialize GPU[%u], declaring it wedged!\n",
249 				j);
250 			intel_gt_set_wedged(gt);
251 		}
252 
253 		if (j == i)
254 			break;
255 	}
256 }
257