1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2019 Intel Corporation
5  */
6 
7 #include "gem/i915_gem_pm.h"
8 #include "gem/i915_gem_ttm_pm.h"
9 #include "gt/intel_gt.h"
10 #include "gt/intel_gt_pm.h"
11 #include "gt/intel_gt_requests.h"
12 
13 #include "i915_driver.h"
14 #include "i915_drv.h"
15 
16 #if defined(CONFIG_X86)
17 #include <asm/smp.h>
18 #else
19 #define wbinvd_on_all_cpus() \
20 	pr_warn(DRIVER_NAME ": Missing cache flush in %s\n", __func__)
21 #endif
22 
23 void i915_gem_suspend(struct drm_i915_private *i915)
24 {
25 	GEM_TRACE("%s\n", dev_name(i915->drm.dev));
26 
27 	intel_wakeref_auto(&to_gt(i915)->ggtt->userfault_wakeref, 0);
28 	flush_workqueue(i915->wq);
29 
30 	/*
31 	 * We have to flush all the executing contexts to main memory so
32 	 * that they can saved in the hibernation image. To ensure the last
33 	 * context image is coherent, we have to switch away from it. That
34 	 * leaves the i915->kernel_context still active when
35 	 * we actually suspend, and its image in memory may not match the GPU
36 	 * state. Fortunately, the kernel_context is disposable and we do
37 	 * not rely on its state.
38 	 */
39 	intel_gt_suspend_prepare(to_gt(i915));
40 
41 	i915_gem_drain_freed_objects(i915);
42 }
43 
44 static int lmem_restore(struct drm_i915_private *i915, u32 flags)
45 {
46 	struct intel_memory_region *mr;
47 	int ret = 0, id;
48 
49 	for_each_memory_region(mr, i915, id) {
50 		if (mr->type == INTEL_MEMORY_LOCAL) {
51 			ret = i915_ttm_restore_region(mr, flags);
52 			if (ret)
53 				break;
54 		}
55 	}
56 
57 	return ret;
58 }
59 
60 static int lmem_suspend(struct drm_i915_private *i915, u32 flags)
61 {
62 	struct intel_memory_region *mr;
63 	int ret = 0, id;
64 
65 	for_each_memory_region(mr, i915, id) {
66 		if (mr->type == INTEL_MEMORY_LOCAL) {
67 			ret = i915_ttm_backup_region(mr, flags);
68 			if (ret)
69 				break;
70 		}
71 	}
72 
73 	return ret;
74 }
75 
76 static void lmem_recover(struct drm_i915_private *i915)
77 {
78 	struct intel_memory_region *mr;
79 	int id;
80 
81 	for_each_memory_region(mr, i915, id)
82 		if (mr->type == INTEL_MEMORY_LOCAL)
83 			i915_ttm_recover_region(mr);
84 }
85 
86 int i915_gem_backup_suspend(struct drm_i915_private *i915)
87 {
88 	int ret;
89 
90 	/* Opportunistically try to evict unpinned objects */
91 	ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU);
92 	if (ret)
93 		goto out_recover;
94 
95 	i915_gem_suspend(i915);
96 
97 	/*
98 	 * More objects may have become unpinned as requests were
99 	 * retired. Now try to evict again. The gt may be wedged here
100 	 * in which case we automatically fall back to memcpy.
101 	 * We allow also backing up pinned objects that have not been
102 	 * marked for early recover, and that may contain, for example,
103 	 * page-tables for the migrate context.
104 	 */
105 	ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU |
106 			   I915_TTM_BACKUP_PINNED);
107 	if (ret)
108 		goto out_recover;
109 
110 	/*
111 	 * Remaining objects are backed up using memcpy once we've stopped
112 	 * using the migrate context.
113 	 */
114 	ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED);
115 	if (ret)
116 		goto out_recover;
117 
118 	return 0;
119 
120 out_recover:
121 	lmem_recover(i915);
122 
123 	return ret;
124 }
125 
126 void i915_gem_suspend_late(struct drm_i915_private *i915)
127 {
128 	struct drm_i915_gem_object *obj;
129 	struct list_head *phases[] = {
130 		&i915->mm.shrink_list,
131 		&i915->mm.purge_list,
132 		NULL
133 	}, **phase;
134 	unsigned long flags;
135 	bool flush = false;
136 
137 	/*
138 	 * Neither the BIOS, ourselves or any other kernel
139 	 * expects the system to be in execlists mode on startup,
140 	 * so we need to reset the GPU back to legacy mode. And the only
141 	 * known way to disable logical contexts is through a GPU reset.
142 	 *
143 	 * So in order to leave the system in a known default configuration,
144 	 * always reset the GPU upon unload and suspend. Afterwards we then
145 	 * clean up the GEM state tracking, flushing off the requests and
146 	 * leaving the system in a known idle state.
147 	 *
148 	 * Note that is of the upmost importance that the GPU is idle and
149 	 * all stray writes are flushed *before* we dismantle the backing
150 	 * storage for the pinned objects.
151 	 *
152 	 * However, since we are uncertain that resetting the GPU on older
153 	 * machines is a good idea, we don't - just in case it leaves the
154 	 * machine in an unusable condition.
155 	 */
156 
157 	intel_gt_suspend_late(to_gt(i915));
158 
159 	spin_lock_irqsave(&i915->mm.obj_lock, flags);
160 	for (phase = phases; *phase; phase++) {
161 		list_for_each_entry(obj, *phase, mm.link) {
162 			if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
163 				flush |= (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0;
164 			__start_cpu_write(obj); /* presume auto-hibernate */
165 		}
166 	}
167 	spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
168 	if (flush)
169 		wbinvd_on_all_cpus();
170 }
171 
172 int i915_gem_freeze(struct drm_i915_private *i915)
173 {
174 	/* Discard all purgeable objects, let userspace recover those as
175 	 * required after resuming.
176 	 */
177 	i915_gem_shrink_all(i915);
178 
179 	return 0;
180 }
181 
182 int i915_gem_freeze_late(struct drm_i915_private *i915)
183 {
184 	struct drm_i915_gem_object *obj;
185 	intel_wakeref_t wakeref;
186 
187 	/*
188 	 * Called just before we write the hibernation image.
189 	 *
190 	 * We need to update the domain tracking to reflect that the CPU
191 	 * will be accessing all the pages to create and restore from the
192 	 * hibernation, and so upon restoration those pages will be in the
193 	 * CPU domain.
194 	 *
195 	 * To make sure the hibernation image contains the latest state,
196 	 * we update that state just before writing out the image.
197 	 *
198 	 * To try and reduce the hibernation image, we manually shrink
199 	 * the objects as well, see i915_gem_freeze()
200 	 */
201 
202 	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
203 		i915_gem_shrink(NULL, i915, -1UL, NULL, ~0);
204 	i915_gem_drain_freed_objects(i915);
205 
206 	wbinvd_on_all_cpus();
207 	list_for_each_entry(obj, &i915->mm.shrink_list, mm.link)
208 		__start_cpu_write(obj);
209 
210 	return 0;
211 }
212 
213 void i915_gem_resume(struct drm_i915_private *i915)
214 {
215 	int ret;
216 
217 	GEM_TRACE("%s\n", dev_name(i915->drm.dev));
218 
219 	ret = lmem_restore(i915, 0);
220 	GEM_WARN_ON(ret);
221 
222 	/*
223 	 * As we didn't flush the kernel context before suspend, we cannot
224 	 * guarantee that the context image is complete. So let's just reset
225 	 * it and start again.
226 	 */
227 	intel_gt_resume(to_gt(i915));
228 
229 	ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU);
230 	GEM_WARN_ON(ret);
231 }
232