1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include "i915_drv.h"
7 #include "intel_display_types.h"
8 #include "intel_dpt.h"
9 #include "intel_fb.h"
10 #include "gt/gen8_ppgtt.h"
11 
12 struct i915_dpt {
13 	struct i915_address_space vm;
14 
15 	struct drm_i915_gem_object *obj;
16 	struct i915_vma *vma;
17 	void __iomem *iomem;
18 };
19 
20 #define i915_is_dpt(vm) ((vm)->is_dpt)
21 
22 static inline struct i915_dpt *
23 i915_vm_to_dpt(struct i915_address_space *vm)
24 {
25 	BUILD_BUG_ON(offsetof(struct i915_dpt, vm));
26 	GEM_BUG_ON(!i915_is_dpt(vm));
27 	return container_of(vm, struct i915_dpt, vm);
28 }
29 
30 #define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT)
31 
32 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
33 {
34 	writeq(pte, addr);
35 }
36 
37 static void dpt_insert_page(struct i915_address_space *vm,
38 			    dma_addr_t addr,
39 			    u64 offset,
40 			    enum i915_cache_level level,
41 			    u32 flags)
42 {
43 	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
44 	gen8_pte_t __iomem *base = dpt->iomem;
45 
46 	gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE,
47 		     vm->pte_encode(addr, level, flags));
48 }
49 
50 static void dpt_insert_entries(struct i915_address_space *vm,
51 			       struct i915_vma *vma,
52 			       enum i915_cache_level level,
53 			       u32 flags)
54 {
55 	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
56 	gen8_pte_t __iomem *base = dpt->iomem;
57 	const gen8_pte_t pte_encode = vm->pte_encode(0, level, flags);
58 	struct sgt_iter sgt_iter;
59 	dma_addr_t addr;
60 	int i;
61 
62 	/*
63 	 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
64 	 * not to allow the user to override access to a read only page.
65 	 */
66 
67 	i = vma->node.start / I915_GTT_PAGE_SIZE;
68 	for_each_sgt_daddr(addr, sgt_iter, vma->pages)
69 		gen8_set_pte(&base[i++], pte_encode | addr);
70 }
71 
72 static void dpt_clear_range(struct i915_address_space *vm,
73 			    u64 start, u64 length)
74 {
75 }
76 
77 static void dpt_bind_vma(struct i915_address_space *vm,
78 			 struct i915_vm_pt_stash *stash,
79 			 struct i915_vma *vma,
80 			 enum i915_cache_level cache_level,
81 			 u32 flags)
82 {
83 	struct drm_i915_gem_object *obj = vma->obj;
84 	u32 pte_flags;
85 
86 	/* Applicable to VLV (gen8+ do not support RO in the GGTT) */
87 	pte_flags = 0;
88 	if (vma->vm->has_read_only && i915_gem_object_is_readonly(obj))
89 		pte_flags |= PTE_READ_ONLY;
90 	if (i915_gem_object_is_lmem(obj))
91 		pte_flags |= PTE_LM;
92 
93 	vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
94 
95 	vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
96 
97 	/*
98 	 * Without aliasing PPGTT there's no difference between
99 	 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
100 	 * upgrade to both bound if we bind either to avoid double-binding.
101 	 */
102 	atomic_or(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND, &vma->flags);
103 }
104 
105 static void dpt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma)
106 {
107 	vm->clear_range(vm, vma->node.start, vma->size);
108 }
109 
110 static void dpt_cleanup(struct i915_address_space *vm)
111 {
112 	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
113 
114 	i915_gem_object_put(dpt->obj);
115 }
116 
117 struct i915_vma *intel_dpt_pin(struct i915_address_space *vm)
118 {
119 	struct drm_i915_private *i915 = vm->i915;
120 	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
121 	intel_wakeref_t wakeref;
122 	struct i915_vma *vma;
123 	void __iomem *iomem;
124 	struct i915_gem_ww_ctx ww;
125 	int err;
126 
127 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
128 	atomic_inc(&i915->gpu_error.pending_fb_pin);
129 
130 	for_i915_gem_ww(&ww, err, true) {
131 		err = i915_gem_object_lock(dpt->obj, &ww);
132 		if (err)
133 			continue;
134 
135 		vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0, 4096,
136 						  HAS_LMEM(i915) ? 0 : PIN_MAPPABLE);
137 		if (IS_ERR(vma)) {
138 			err = PTR_ERR(vma);
139 			continue;
140 		}
141 
142 		iomem = i915_vma_pin_iomap(vma);
143 		i915_vma_unpin(vma);
144 
145 		if (IS_ERR(iomem)) {
146 			err = PTR_ERR(iomem);
147 			continue;
148 		}
149 
150 		dpt->vma = vma;
151 		dpt->iomem = iomem;
152 
153 		i915_vma_get(vma);
154 	}
155 
156 	atomic_dec(&i915->gpu_error.pending_fb_pin);
157 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
158 
159 	return err ? ERR_PTR(err) : vma;
160 }
161 
162 void intel_dpt_unpin(struct i915_address_space *vm)
163 {
164 	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
165 
166 	i915_vma_unpin_iomap(dpt->vma);
167 	i915_vma_put(dpt->vma);
168 }
169 
170 /**
171  * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume
172  * @i915: device instance
173  *
174  * Restore the memory mapping during system resume for all framebuffers which
175  * are mapped to HW via a GGTT->DPT page table. The content of these page
176  * tables are not stored in the hibernation image during S4 and S3RST->S4
177  * transitions, so here we reprogram the PTE entries in those tables.
178  *
179  * This function must be called after the mappings in GGTT have been restored calling
180  * i915_ggtt_resume().
181  */
182 void intel_dpt_resume(struct drm_i915_private *i915)
183 {
184 	struct drm_framebuffer *drm_fb;
185 
186 	if (!HAS_DISPLAY(i915))
187 		return;
188 
189 	mutex_lock(&i915->drm.mode_config.fb_lock);
190 	drm_for_each_fb(drm_fb, &i915->drm) {
191 		struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
192 
193 		if (fb->dpt_vm)
194 			i915_ggtt_resume_vm(fb->dpt_vm);
195 	}
196 	mutex_unlock(&i915->drm.mode_config.fb_lock);
197 }
198 
199 /**
200  * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend
201  * @i915: device instance
202  *
203  * Suspend the memory mapping during system suspend for all framebuffers which
204  * are mapped to HW via a GGTT->DPT page table.
205  *
206  * This function must be called before the mappings in GGTT are suspended calling
207  * i915_ggtt_suspend().
208  */
209 void intel_dpt_suspend(struct drm_i915_private *i915)
210 {
211 	struct drm_framebuffer *drm_fb;
212 
213 	if (!HAS_DISPLAY(i915))
214 		return;
215 
216 	mutex_lock(&i915->drm.mode_config.fb_lock);
217 
218 	drm_for_each_fb(drm_fb, &i915->drm) {
219 		struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
220 
221 		if (fb->dpt_vm)
222 			i915_ggtt_suspend_vm(fb->dpt_vm);
223 	}
224 
225 	mutex_unlock(&i915->drm.mode_config.fb_lock);
226 }
227 
228 struct i915_address_space *
229 intel_dpt_create(struct intel_framebuffer *fb)
230 {
231 	struct drm_gem_object *obj = &intel_fb_obj(&fb->base)->base;
232 	struct drm_i915_private *i915 = to_i915(obj->dev);
233 	struct drm_i915_gem_object *dpt_obj;
234 	struct i915_address_space *vm;
235 	struct i915_dpt *dpt;
236 	size_t size;
237 	int ret;
238 
239 	if (intel_fb_needs_pot_stride_remap(fb))
240 		size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped);
241 	else
242 		size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE);
243 
244 	size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE);
245 
246 	if (HAS_LMEM(i915))
247 		dpt_obj = i915_gem_object_create_lmem(i915, size, 0);
248 	else
249 		dpt_obj = i915_gem_object_create_stolen(i915, size);
250 	if (IS_ERR(dpt_obj))
251 		return ERR_CAST(dpt_obj);
252 
253 	ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE);
254 	if (ret) {
255 		i915_gem_object_put(dpt_obj);
256 		return ERR_PTR(ret);
257 	}
258 
259 	dpt = kzalloc(sizeof(*dpt), GFP_KERNEL);
260 	if (!dpt) {
261 		i915_gem_object_put(dpt_obj);
262 		return ERR_PTR(-ENOMEM);
263 	}
264 
265 	vm = &dpt->vm;
266 
267 	vm->gt = to_gt(i915);
268 	vm->i915 = i915;
269 	vm->dma = i915->drm.dev;
270 	vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
271 	vm->is_dpt = true;
272 
273 	i915_address_space_init(vm, VM_CLASS_DPT);
274 
275 	vm->insert_page = dpt_insert_page;
276 	vm->clear_range = dpt_clear_range;
277 	vm->insert_entries = dpt_insert_entries;
278 	vm->cleanup = dpt_cleanup;
279 
280 	vm->vma_ops.bind_vma    = dpt_bind_vma;
281 	vm->vma_ops.unbind_vma  = dpt_unbind_vma;
282 
283 	vm->pte_encode = gen8_ggtt_pte_encode;
284 
285 	dpt->obj = dpt_obj;
286 
287 	return &dpt->vm;
288 }
289 
290 void intel_dpt_destroy(struct i915_address_space *vm)
291 {
292 	struct i915_dpt *dpt = i915_vm_to_dpt(vm);
293 
294 	i915_vm_close(&dpt->vm);
295 }
296