1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include <drm/ttm/ttm_placement.h>
7 #include <drm/ttm/ttm_tt.h>
8 
9 #include "i915_drv.h"
10 #include "intel_memory_region.h"
11 #include "intel_region_ttm.h"
12 
13 #include "gem/i915_gem_region.h"
14 #include "gem/i915_gem_ttm.h"
15 #include "gem/i915_gem_ttm_move.h"
16 #include "gem/i915_gem_ttm_pm.h"
17 
18 /**
19  * i915_ttm_backup_free - Free any backup attached to this object
20  * @obj: The object whose backup is to be freed.
21  */
22 void i915_ttm_backup_free(struct drm_i915_gem_object *obj)
23 {
24 	if (obj->ttm.backup) {
25 		i915_gem_object_put(obj->ttm.backup);
26 		obj->ttm.backup = NULL;
27 	}
28 }
29 
30 /**
31  * struct i915_gem_ttm_pm_apply - Apply-to-region subclass for restore
32  * @base: The i915_gem_apply_to_region we derive from.
33  * @allow_gpu: Whether using the gpu blitter is allowed.
34  * @backup_pinned: On backup, backup also pinned objects.
35  */
36 struct i915_gem_ttm_pm_apply {
37 	struct i915_gem_apply_to_region base;
38 	bool allow_gpu : 1;
39 	bool backup_pinned : 1;
40 };
41 
42 static int i915_ttm_backup(struct i915_gem_apply_to_region *apply,
43 			   struct drm_i915_gem_object *obj)
44 {
45 	struct i915_gem_ttm_pm_apply *pm_apply =
46 		container_of(apply, typeof(*pm_apply), base);
47 	struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
48 	struct ttm_buffer_object *backup_bo;
49 	struct drm_i915_private *i915 =
50 		container_of(bo->bdev, typeof(*i915), bdev);
51 	struct drm_i915_gem_object *backup;
52 	struct ttm_operation_ctx ctx = {};
53 	unsigned int flags;
54 	int err = 0;
55 
56 	if (!i915_ttm_cpu_maps_iomem(bo->resource) || obj->ttm.backup)
57 		return 0;
58 
59 	if (pm_apply->allow_gpu && i915_gem_object_evictable(obj))
60 		return ttm_bo_validate(bo, i915_ttm_sys_placement(), &ctx);
61 
62 	if (!pm_apply->backup_pinned ||
63 	    (pm_apply->allow_gpu && (obj->flags & I915_BO_ALLOC_PM_EARLY)))
64 		return 0;
65 
66 	if (obj->flags & I915_BO_ALLOC_PM_VOLATILE)
67 		return 0;
68 
69 	/*
70 	 * It seems that we might have some framebuffers still pinned at this
71 	 * stage, but for such objects we might also need to deal with the CCS
72 	 * aux state. Make sure we force the save/restore of the CCS state,
73 	 * otherwise we might observe display corruption, when returning from
74 	 * suspend.
75 	 */
76 	flags = 0;
77 	if (i915_gem_object_needs_ccs_pages(obj)) {
78 		WARN_ON_ONCE(!i915_gem_object_is_framebuffer(obj));
79 		WARN_ON_ONCE(!pm_apply->allow_gpu);
80 
81 		flags = I915_BO_ALLOC_CCS_AUX;
82 	}
83 	backup = i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_SMEM],
84 					       obj->base.size, 0, flags);
85 	if (IS_ERR(backup))
86 		return PTR_ERR(backup);
87 
88 	err = i915_gem_object_lock(backup, apply->ww);
89 	if (err)
90 		goto out_no_lock;
91 
92 	backup_bo = i915_gem_to_ttm(backup);
93 	err = ttm_tt_populate(backup_bo->bdev, backup_bo->ttm, &ctx);
94 	if (err)
95 		goto out_no_populate;
96 
97 	err = i915_gem_obj_copy_ttm(backup, obj, pm_apply->allow_gpu, false);
98 	if (err) {
99 		drm_err(&i915->drm,
100 			"Unable to copy from device to system memory, err:%pe\n",
101 			ERR_PTR(err));
102 		goto out_no_populate;
103 	}
104 	ttm_bo_wait_ctx(backup_bo, &ctx);
105 
106 	obj->ttm.backup = backup;
107 	return 0;
108 
109 out_no_populate:
110 	i915_gem_ww_unlock_single(backup);
111 out_no_lock:
112 	i915_gem_object_put(backup);
113 
114 	return err;
115 }
116 
117 static int i915_ttm_recover(struct i915_gem_apply_to_region *apply,
118 			    struct drm_i915_gem_object *obj)
119 {
120 	i915_ttm_backup_free(obj);
121 	return 0;
122 }
123 
124 /**
125  * i915_ttm_recover_region - Free the backup of all objects of a region
126  * @mr: The memory region
127  *
128  * Checks all objects of a region if there is backup attached and if so
129  * frees that backup. Typically this is called to recover after a partially
130  * performed backup.
131  */
132 void i915_ttm_recover_region(struct intel_memory_region *mr)
133 {
134 	static const struct i915_gem_apply_to_region_ops recover_ops = {
135 		.process_obj = i915_ttm_recover,
136 	};
137 	struct i915_gem_apply_to_region apply = {.ops = &recover_ops};
138 	int ret;
139 
140 	ret = i915_gem_process_region(mr, &apply);
141 	GEM_WARN_ON(ret);
142 }
143 
144 /**
145  * i915_ttm_backup_region - Back up all objects of a region to smem.
146  * @mr: The memory region
147  * @flags: TTM backup flags
148  *
149  * Loops over all objects of a region and either evicts them if they are
150  * evictable or backs them up using a backup object if they are pinned.
151  *
152  * Return: Zero on success. Negative error code on error.
153  */
154 int i915_ttm_backup_region(struct intel_memory_region *mr, u32 flags)
155 {
156 	static const struct i915_gem_apply_to_region_ops backup_ops = {
157 		.process_obj = i915_ttm_backup,
158 	};
159 	struct i915_gem_ttm_pm_apply pm_apply = {
160 		.base = {.ops = &backup_ops},
161 		.allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU,
162 		.backup_pinned = flags & I915_TTM_BACKUP_PINNED,
163 	};
164 
165 	return i915_gem_process_region(mr, &pm_apply.base);
166 }
167 
168 static int i915_ttm_restore(struct i915_gem_apply_to_region *apply,
169 			    struct drm_i915_gem_object *obj)
170 {
171 	struct i915_gem_ttm_pm_apply *pm_apply =
172 		container_of(apply, typeof(*pm_apply), base);
173 	struct drm_i915_gem_object *backup = obj->ttm.backup;
174 	struct ttm_buffer_object *backup_bo = i915_gem_to_ttm(backup);
175 	struct ttm_operation_ctx ctx = {};
176 	int err;
177 
178 	if (!backup)
179 		return 0;
180 
181 	if (!pm_apply->allow_gpu && !(obj->flags & I915_BO_ALLOC_PM_EARLY))
182 		return 0;
183 
184 	err = i915_gem_object_lock(backup, apply->ww);
185 	if (err)
186 		return err;
187 
188 	/* Content may have been swapped. */
189 	if (!backup_bo->resource)
190 		err = ttm_bo_validate(backup_bo, i915_ttm_sys_placement(), &ctx);
191 	if (!err)
192 		err = ttm_tt_populate(backup_bo->bdev, backup_bo->ttm, &ctx);
193 	if (!err) {
194 		err = i915_gem_obj_copy_ttm(obj, backup, pm_apply->allow_gpu,
195 					    false);
196 		GEM_WARN_ON(err);
197 		ttm_bo_wait_ctx(backup_bo, &ctx);
198 
199 		obj->ttm.backup = NULL;
200 		err = 0;
201 	}
202 
203 	i915_gem_ww_unlock_single(backup);
204 
205 	if (!err)
206 		i915_gem_object_put(backup);
207 
208 	return err;
209 }
210 
211 /**
212  * i915_ttm_restore_region - Restore backed-up objects of a region from smem.
213  * @mr: The memory region
214  * @flags: TTM backup flags
215  *
216  * Loops over all objects of a region and if they are backed-up, restores
217  * them from smem.
218  *
219  * Return: Zero on success. Negative error code on error.
220  */
221 int i915_ttm_restore_region(struct intel_memory_region *mr, u32 flags)
222 {
223 	static const struct i915_gem_apply_to_region_ops restore_ops = {
224 		.process_obj = i915_ttm_restore,
225 	};
226 	struct i915_gem_ttm_pm_apply pm_apply = {
227 		.base = {.ops = &restore_ops},
228 		.allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU,
229 	};
230 
231 	return i915_gem_process_region(mr, &pm_apply.base);
232 }
233