1 /*
2  * Copyright © 2017 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  */
24 
25 #include "display/intel_frontbuffer.h"
26 #include "gt/intel_gt.h"
27 #include "i915_drv.h"
28 #include "i915_gem_clflush.h"
29 #include "i915_gem_context.h"
30 #include "i915_gem_object.h"
31 #include "i915_globals.h"
32 #include "i915_trace.h"
33 
34 static struct i915_global_object {
35 	struct i915_global base;
36 	struct kmem_cache *slab_objects;
37 } global;
38 
39 struct drm_i915_gem_object *i915_gem_object_alloc(void)
40 {
41 	return kmem_cache_zalloc(global.slab_objects, GFP_KERNEL);
42 }
43 
44 void i915_gem_object_free(struct drm_i915_gem_object *obj)
45 {
46 	return kmem_cache_free(global.slab_objects, obj);
47 }
48 
49 void i915_gem_object_init(struct drm_i915_gem_object *obj,
50 			  const struct drm_i915_gem_object_ops *ops,
51 			  struct lock_class_key *key)
52 {
53 	__mutex_init(&obj->mm.lock, "obj->mm.lock", key);
54 
55 	spin_lock_init(&obj->vma.lock);
56 	INIT_LIST_HEAD(&obj->vma.list);
57 
58 	INIT_LIST_HEAD(&obj->mm.link);
59 
60 	INIT_LIST_HEAD(&obj->lut_list);
61 
62 	init_rcu_head(&obj->rcu);
63 
64 	obj->ops = ops;
65 
66 	obj->mm.madv = I915_MADV_WILLNEED;
67 	INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN);
68 	mutex_init(&obj->mm.get_page.lock);
69 }
70 
71 /**
72  * Mark up the object's coherency levels for a given cache_level
73  * @obj: #drm_i915_gem_object
74  * @cache_level: cache level
75  */
76 void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
77 					 unsigned int cache_level)
78 {
79 	obj->cache_level = cache_level;
80 
81 	if (cache_level != I915_CACHE_NONE)
82 		obj->cache_coherent = (I915_BO_CACHE_COHERENT_FOR_READ |
83 				       I915_BO_CACHE_COHERENT_FOR_WRITE);
84 	else if (HAS_LLC(to_i915(obj->base.dev)))
85 		obj->cache_coherent = I915_BO_CACHE_COHERENT_FOR_READ;
86 	else
87 		obj->cache_coherent = 0;
88 
89 	obj->cache_dirty =
90 		!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE);
91 }
92 
93 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
94 {
95 	struct drm_i915_gem_object *obj = to_intel_bo(gem);
96 	struct drm_i915_file_private *fpriv = file->driver_priv;
97 	struct i915_lut_handle *lut, *ln;
98 	LIST_HEAD(close);
99 
100 	i915_gem_object_lock(obj);
101 	list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) {
102 		struct i915_gem_context *ctx = lut->ctx;
103 
104 		if (ctx->file_priv != fpriv)
105 			continue;
106 
107 		i915_gem_context_get(ctx);
108 		list_move(&lut->obj_link, &close);
109 	}
110 	i915_gem_object_unlock(obj);
111 
112 	list_for_each_entry_safe(lut, ln, &close, obj_link) {
113 		struct i915_gem_context *ctx = lut->ctx;
114 		struct i915_vma *vma;
115 
116 		/*
117 		 * We allow the process to have multiple handles to the same
118 		 * vma, in the same fd namespace, by virtue of flink/open.
119 		 */
120 
121 		mutex_lock(&ctx->mutex);
122 		vma = radix_tree_delete(&ctx->handles_vma, lut->handle);
123 		if (vma) {
124 			GEM_BUG_ON(vma->obj != obj);
125 			GEM_BUG_ON(!atomic_read(&vma->open_count));
126 			if (atomic_dec_and_test(&vma->open_count) &&
127 			    !i915_vma_is_ggtt(vma))
128 				i915_vma_close(vma);
129 		}
130 		mutex_unlock(&ctx->mutex);
131 
132 		i915_gem_context_put(lut->ctx);
133 		i915_lut_handle_free(lut);
134 		i915_gem_object_put(obj);
135 	}
136 }
137 
138 static void __i915_gem_free_object_rcu(struct rcu_head *head)
139 {
140 	struct drm_i915_gem_object *obj =
141 		container_of(head, typeof(*obj), rcu);
142 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
143 
144 	dma_resv_fini(&obj->base._resv);
145 	i915_gem_object_free(obj);
146 
147 	GEM_BUG_ON(!atomic_read(&i915->mm.free_count));
148 	atomic_dec(&i915->mm.free_count);
149 }
150 
151 static void __i915_gem_free_objects(struct drm_i915_private *i915,
152 				    struct llist_node *freed)
153 {
154 	struct drm_i915_gem_object *obj, *on;
155 	intel_wakeref_t wakeref;
156 
157 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
158 	llist_for_each_entry_safe(obj, on, freed, freed) {
159 		trace_i915_gem_object_destroy(obj);
160 
161 		if (!list_empty(&obj->vma.list)) {
162 			struct i915_vma *vma;
163 
164 			/*
165 			 * Note that the vma keeps an object reference while
166 			 * it is active, so it *should* not sleep while we
167 			 * destroy it. Our debug code errs insits it *might*.
168 			 * For the moment, play along.
169 			 */
170 			spin_lock(&obj->vma.lock);
171 			while ((vma = list_first_entry_or_null(&obj->vma.list,
172 							       struct i915_vma,
173 							       obj_link))) {
174 				GEM_BUG_ON(vma->obj != obj);
175 				spin_unlock(&obj->vma.lock);
176 
177 				i915_vma_destroy(vma);
178 
179 				spin_lock(&obj->vma.lock);
180 			}
181 			spin_unlock(&obj->vma.lock);
182 		}
183 
184 		GEM_BUG_ON(atomic_read(&obj->bind_count));
185 		GEM_BUG_ON(obj->userfault_count);
186 		GEM_BUG_ON(!list_empty(&obj->lut_list));
187 
188 		atomic_set(&obj->mm.pages_pin_count, 0);
189 		__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
190 		GEM_BUG_ON(i915_gem_object_has_pages(obj));
191 		bitmap_free(obj->bit_17);
192 
193 		if (obj->base.import_attach)
194 			drm_prime_gem_destroy(&obj->base, NULL);
195 
196 		drm_gem_free_mmap_offset(&obj->base);
197 
198 		if (obj->ops->release)
199 			obj->ops->release(obj);
200 
201 		/* But keep the pointer alive for RCU-protected lookups */
202 		call_rcu(&obj->rcu, __i915_gem_free_object_rcu);
203 	}
204 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
205 }
206 
207 void i915_gem_flush_free_objects(struct drm_i915_private *i915)
208 {
209 	struct llist_node *freed = llist_del_all(&i915->mm.free_list);
210 
211 	if (unlikely(freed))
212 		__i915_gem_free_objects(i915, freed);
213 }
214 
215 static void __i915_gem_free_work(struct work_struct *work)
216 {
217 	struct drm_i915_private *i915 =
218 		container_of(work, struct drm_i915_private, mm.free_work);
219 
220 	i915_gem_flush_free_objects(i915);
221 }
222 
223 void i915_gem_free_object(struct drm_gem_object *gem_obj)
224 {
225 	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
226 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
227 
228 	GEM_BUG_ON(i915_gem_object_is_framebuffer(obj));
229 
230 	/*
231 	 * Before we free the object, make sure any pure RCU-only
232 	 * read-side critical sections are complete, e.g.
233 	 * i915_gem_busy_ioctl(). For the corresponding synchronized
234 	 * lookup see i915_gem_object_lookup_rcu().
235 	 */
236 	atomic_inc(&i915->mm.free_count);
237 
238 	/*
239 	 * This serializes freeing with the shrinker. Since the free
240 	 * is delayed, first by RCU then by the workqueue, we want the
241 	 * shrinker to be able to free pages of unreferenced objects,
242 	 * or else we may oom whilst there are plenty of deferred
243 	 * freed objects.
244 	 */
245 	i915_gem_object_make_unshrinkable(obj);
246 
247 	/*
248 	 * Since we require blocking on struct_mutex to unbind the freed
249 	 * object from the GPU before releasing resources back to the
250 	 * system, we can not do that directly from the RCU callback (which may
251 	 * be a softirq context), but must instead then defer that work onto a
252 	 * kthread. We use the RCU callback rather than move the freed object
253 	 * directly onto the work queue so that we can mix between using the
254 	 * worker and performing frees directly from subsequent allocations for
255 	 * crude but effective memory throttling.
256 	 */
257 	if (llist_add(&obj->freed, &i915->mm.free_list))
258 		queue_work(i915->wq, &i915->mm.free_work);
259 }
260 
261 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
262 {
263 	return !(obj->cache_level == I915_CACHE_NONE ||
264 		 obj->cache_level == I915_CACHE_WT);
265 }
266 
267 void
268 i915_gem_object_flush_write_domain(struct drm_i915_gem_object *obj,
269 				   unsigned int flush_domains)
270 {
271 	struct i915_vma *vma;
272 
273 	assert_object_held(obj);
274 
275 	if (!(obj->write_domain & flush_domains))
276 		return;
277 
278 	switch (obj->write_domain) {
279 	case I915_GEM_DOMAIN_GTT:
280 		for_each_ggtt_vma(vma, obj)
281 			intel_gt_flush_ggtt_writes(vma->vm->gt);
282 
283 		i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
284 
285 		for_each_ggtt_vma(vma, obj) {
286 			if (vma->iomap)
287 				continue;
288 
289 			i915_vma_unset_ggtt_write(vma);
290 		}
291 
292 		break;
293 
294 	case I915_GEM_DOMAIN_WC:
295 		wmb();
296 		break;
297 
298 	case I915_GEM_DOMAIN_CPU:
299 		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
300 		break;
301 
302 	case I915_GEM_DOMAIN_RENDER:
303 		if (gpu_write_needs_clflush(obj))
304 			obj->cache_dirty = true;
305 		break;
306 	}
307 
308 	obj->write_domain = 0;
309 }
310 
311 void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj,
312 					 enum fb_op_origin origin)
313 {
314 	struct intel_frontbuffer *front;
315 
316 	front = __intel_frontbuffer_get(obj);
317 	if (front) {
318 		intel_frontbuffer_flush(front, origin);
319 		intel_frontbuffer_put(front);
320 	}
321 }
322 
323 void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj,
324 					      enum fb_op_origin origin)
325 {
326 	struct intel_frontbuffer *front;
327 
328 	front = __intel_frontbuffer_get(obj);
329 	if (front) {
330 		intel_frontbuffer_invalidate(front, origin);
331 		intel_frontbuffer_put(front);
332 	}
333 }
334 
335 void i915_gem_init__objects(struct drm_i915_private *i915)
336 {
337 	INIT_WORK(&i915->mm.free_work, __i915_gem_free_work);
338 }
339 
340 static void i915_global_objects_shrink(void)
341 {
342 	kmem_cache_shrink(global.slab_objects);
343 }
344 
345 static void i915_global_objects_exit(void)
346 {
347 	kmem_cache_destroy(global.slab_objects);
348 }
349 
350 static struct i915_global_object global = { {
351 	.shrink = i915_global_objects_shrink,
352 	.exit = i915_global_objects_exit,
353 } };
354 
355 int __init i915_global_objects_init(void)
356 {
357 	global.slab_objects =
358 		KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);
359 	if (!global.slab_objects)
360 		return -ENOMEM;
361 
362 	i915_global_register(&global.base);
363 	return 0;
364 }
365 
366 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
367 #include "selftests/huge_gem_object.c"
368 #include "selftests/huge_pages.c"
369 #include "selftests/i915_gem_object.c"
370 #include "selftests/i915_gem_coherency.c"
371 #endif
372