1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2016 Intel Corporation
5  */
6 
7 #ifndef __I915_GEM_OBJECT_H__
8 #define __I915_GEM_OBJECT_H__
9 
10 #include <drm/drm_gem.h>
11 #include <drm/drm_file.h>
12 #include <drm/drm_device.h>
13 
14 #include <drm/i915_drm.h>
15 
16 #include "i915_gem_object_types.h"
17 
18 #include "i915_gem_gtt.h"
19 
20 void i915_gem_init__objects(struct drm_i915_private *i915);
21 
22 struct drm_i915_gem_object *i915_gem_object_alloc(void);
23 void i915_gem_object_free(struct drm_i915_gem_object *obj);
24 
25 void i915_gem_object_init(struct drm_i915_gem_object *obj,
26 			  const struct drm_i915_gem_object_ops *ops);
27 struct drm_i915_gem_object *
28 i915_gem_object_create_shmem(struct drm_i915_private *i915, u64 size);
29 struct drm_i915_gem_object *
30 i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915,
31 				       const void *data, size_t size);
32 
33 extern const struct drm_i915_gem_object_ops i915_gem_shmem_ops;
34 void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
35 				     struct sg_table *pages,
36 				     bool needs_clflush);
37 
38 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align);
39 
40 void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file);
41 void i915_gem_free_object(struct drm_gem_object *obj);
42 
43 void i915_gem_flush_free_objects(struct drm_i915_private *i915);
44 
45 struct sg_table *
46 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj);
47 void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
48 
49 /**
50  * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle
51  * @filp: DRM file private date
52  * @handle: userspace handle
53  *
54  * Returns:
55  *
56  * A pointer to the object named by the handle if such exists on @filp, NULL
57  * otherwise. This object is only valid whilst under the RCU read lock, and
58  * note carefully the object may be in the process of being destroyed.
59  */
60 static inline struct drm_i915_gem_object *
61 i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle)
62 {
63 #ifdef CONFIG_LOCKDEP
64 	WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map));
65 #endif
66 	return idr_find(&file->object_idr, handle);
67 }
68 
69 static inline struct drm_i915_gem_object *
70 i915_gem_object_lookup(struct drm_file *file, u32 handle)
71 {
72 	struct drm_i915_gem_object *obj;
73 
74 	rcu_read_lock();
75 	obj = i915_gem_object_lookup_rcu(file, handle);
76 	if (obj && !kref_get_unless_zero(&obj->base.refcount))
77 		obj = NULL;
78 	rcu_read_unlock();
79 
80 	return obj;
81 }
82 
83 __deprecated
84 struct drm_gem_object *
85 drm_gem_object_lookup(struct drm_file *file, u32 handle);
86 
87 __attribute__((nonnull))
88 static inline struct drm_i915_gem_object *
89 i915_gem_object_get(struct drm_i915_gem_object *obj)
90 {
91 	drm_gem_object_get(&obj->base);
92 	return obj;
93 }
94 
95 __attribute__((nonnull))
96 static inline void
97 i915_gem_object_put(struct drm_i915_gem_object *obj)
98 {
99 	__drm_gem_object_put(&obj->base);
100 }
101 
102 #define assert_object_held(obj) dma_resv_assert_held((obj)->base.resv)
103 
104 static inline void i915_gem_object_lock(struct drm_i915_gem_object *obj)
105 {
106 	dma_resv_lock(obj->base.resv, NULL);
107 }
108 
109 static inline int
110 i915_gem_object_lock_interruptible(struct drm_i915_gem_object *obj)
111 {
112 	return dma_resv_lock_interruptible(obj->base.resv, NULL);
113 }
114 
115 static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj)
116 {
117 	dma_resv_unlock(obj->base.resv);
118 }
119 
120 struct dma_fence *
121 i915_gem_object_lock_fence(struct drm_i915_gem_object *obj);
122 void i915_gem_object_unlock_fence(struct drm_i915_gem_object *obj,
123 				  struct dma_fence *fence);
124 
125 static inline void
126 i915_gem_object_set_readonly(struct drm_i915_gem_object *obj)
127 {
128 	obj->base.vma_node.readonly = true;
129 }
130 
131 static inline bool
132 i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj)
133 {
134 	return obj->base.vma_node.readonly;
135 }
136 
137 static inline bool
138 i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
139 {
140 	return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE;
141 }
142 
143 static inline bool
144 i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj)
145 {
146 	return obj->ops->flags & I915_GEM_OBJECT_IS_SHRINKABLE;
147 }
148 
149 static inline bool
150 i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj)
151 {
152 	return obj->ops->flags & I915_GEM_OBJECT_IS_PROXY;
153 }
154 
155 static inline bool
156 i915_gem_object_needs_async_cancel(const struct drm_i915_gem_object *obj)
157 {
158 	return obj->ops->flags & I915_GEM_OBJECT_ASYNC_CANCEL;
159 }
160 
161 static inline bool
162 i915_gem_object_is_framebuffer(const struct drm_i915_gem_object *obj)
163 {
164 	return READ_ONCE(obj->frontbuffer);
165 }
166 
167 static inline unsigned int
168 i915_gem_object_get_tiling(const struct drm_i915_gem_object *obj)
169 {
170 	return obj->tiling_and_stride & TILING_MASK;
171 }
172 
173 static inline bool
174 i915_gem_object_is_tiled(const struct drm_i915_gem_object *obj)
175 {
176 	return i915_gem_object_get_tiling(obj) != I915_TILING_NONE;
177 }
178 
179 static inline unsigned int
180 i915_gem_object_get_stride(const struct drm_i915_gem_object *obj)
181 {
182 	return obj->tiling_and_stride & STRIDE_MASK;
183 }
184 
185 static inline unsigned int
186 i915_gem_tile_height(unsigned int tiling)
187 {
188 	GEM_BUG_ON(!tiling);
189 	return tiling == I915_TILING_Y ? 32 : 8;
190 }
191 
192 static inline unsigned int
193 i915_gem_object_get_tile_height(const struct drm_i915_gem_object *obj)
194 {
195 	return i915_gem_tile_height(i915_gem_object_get_tiling(obj));
196 }
197 
198 static inline unsigned int
199 i915_gem_object_get_tile_row_size(const struct drm_i915_gem_object *obj)
200 {
201 	return (i915_gem_object_get_stride(obj) *
202 		i915_gem_object_get_tile_height(obj));
203 }
204 
205 int i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
206 			       unsigned int tiling, unsigned int stride);
207 
208 struct scatterlist *
209 i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
210 		       unsigned int n, unsigned int *offset);
211 
212 struct page *
213 i915_gem_object_get_page(struct drm_i915_gem_object *obj,
214 			 unsigned int n);
215 
216 struct page *
217 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
218 			       unsigned int n);
219 
220 dma_addr_t
221 i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
222 				    unsigned long n,
223 				    unsigned int *len);
224 
225 dma_addr_t
226 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
227 				unsigned long n);
228 
229 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
230 				 struct sg_table *pages,
231 				 unsigned int sg_page_sizes);
232 
233 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
234 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
235 
236 static inline int __must_check
237 i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
238 {
239 	might_lock(&obj->mm.lock);
240 
241 	if (atomic_inc_not_zero(&obj->mm.pages_pin_count))
242 		return 0;
243 
244 	return __i915_gem_object_get_pages(obj);
245 }
246 
247 static inline bool
248 i915_gem_object_has_pages(struct drm_i915_gem_object *obj)
249 {
250 	return !IS_ERR_OR_NULL(READ_ONCE(obj->mm.pages));
251 }
252 
253 static inline void
254 __i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
255 {
256 	GEM_BUG_ON(!i915_gem_object_has_pages(obj));
257 
258 	atomic_inc(&obj->mm.pages_pin_count);
259 }
260 
261 static inline bool
262 i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj)
263 {
264 	return atomic_read(&obj->mm.pages_pin_count);
265 }
266 
267 static inline void
268 __i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
269 {
270 	GEM_BUG_ON(!i915_gem_object_has_pages(obj));
271 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
272 
273 	atomic_dec(&obj->mm.pages_pin_count);
274 }
275 
276 static inline void
277 i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
278 {
279 	__i915_gem_object_unpin_pages(obj);
280 }
281 
282 enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock/struct_mutex */
283 	I915_MM_NORMAL = 0,
284 	I915_MM_SHRINKER /* called "recursively" from direct-reclaim-esque */
285 };
286 
287 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj,
288 				enum i915_mm_subclass subclass);
289 void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
290 void i915_gem_object_writeback(struct drm_i915_gem_object *obj);
291 
292 enum i915_map_type {
293 	I915_MAP_WB = 0,
294 	I915_MAP_WC,
295 #define I915_MAP_OVERRIDE BIT(31)
296 	I915_MAP_FORCE_WB = I915_MAP_WB | I915_MAP_OVERRIDE,
297 	I915_MAP_FORCE_WC = I915_MAP_WC | I915_MAP_OVERRIDE,
298 };
299 
300 /**
301  * i915_gem_object_pin_map - return a contiguous mapping of the entire object
302  * @obj: the object to map into kernel address space
303  * @type: the type of mapping, used to select pgprot_t
304  *
305  * Calls i915_gem_object_pin_pages() to prevent reaping of the object's
306  * pages and then returns a contiguous mapping of the backing storage into
307  * the kernel address space. Based on the @type of mapping, the PTE will be
308  * set to either WriteBack or WriteCombine (via pgprot_t).
309  *
310  * The caller is responsible for calling i915_gem_object_unpin_map() when the
311  * mapping is no longer required.
312  *
313  * Returns the pointer through which to access the mapped object, or an
314  * ERR_PTR() on error.
315  */
316 void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
317 					   enum i915_map_type type);
318 
319 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
320 				 unsigned long offset,
321 				 unsigned long size);
322 static inline void i915_gem_object_flush_map(struct drm_i915_gem_object *obj)
323 {
324 	__i915_gem_object_flush_map(obj, 0, obj->base.size);
325 }
326 
327 /**
328  * i915_gem_object_unpin_map - releases an earlier mapping
329  * @obj: the object to unmap
330  *
331  * After pinning the object and mapping its pages, once you are finished
332  * with your access, call i915_gem_object_unpin_map() to release the pin
333  * upon the mapping. Once the pin count reaches zero, that mapping may be
334  * removed.
335  */
336 static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj)
337 {
338 	i915_gem_object_unpin_pages(obj);
339 }
340 
341 void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj);
342 void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj);
343 
344 void
345 i915_gem_object_flush_write_domain(struct drm_i915_gem_object *obj,
346 				   unsigned int flush_domains);
347 
348 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
349 				 unsigned int *needs_clflush);
350 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
351 				  unsigned int *needs_clflush);
352 #define CLFLUSH_BEFORE	BIT(0)
353 #define CLFLUSH_AFTER	BIT(1)
354 #define CLFLUSH_FLAGS	(CLFLUSH_BEFORE | CLFLUSH_AFTER)
355 
356 static inline void
357 i915_gem_object_finish_access(struct drm_i915_gem_object *obj)
358 {
359 	i915_gem_object_unpin_pages(obj);
360 	i915_gem_object_unlock(obj);
361 }
362 
363 static inline struct intel_engine_cs *
364 i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj)
365 {
366 	struct intel_engine_cs *engine = NULL;
367 	struct dma_fence *fence;
368 
369 	rcu_read_lock();
370 	fence = dma_resv_get_excl_rcu(obj->base.resv);
371 	rcu_read_unlock();
372 
373 	if (fence && dma_fence_is_i915(fence) && !dma_fence_is_signaled(fence))
374 		engine = to_request(fence)->engine;
375 	dma_fence_put(fence);
376 
377 	return engine;
378 }
379 
380 void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
381 					 unsigned int cache_level);
382 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj);
383 
384 int __must_check
385 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write);
386 int __must_check
387 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write);
388 int __must_check
389 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write);
390 struct i915_vma * __must_check
391 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
392 				     u32 alignment,
393 				     const struct i915_ggtt_view *view,
394 				     unsigned int flags);
395 void i915_gem_object_unpin_from_display_plane(struct i915_vma *vma);
396 
397 void i915_gem_object_make_unshrinkable(struct drm_i915_gem_object *obj);
398 void i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj);
399 void i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj);
400 
401 static inline bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
402 {
403 	if (obj->cache_dirty)
404 		return false;
405 
406 	if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
407 		return true;
408 
409 	return obj->pin_global; /* currently in use by HW, keep flushed */
410 }
411 
412 static inline void __start_cpu_write(struct drm_i915_gem_object *obj)
413 {
414 	obj->read_domains = I915_GEM_DOMAIN_CPU;
415 	obj->write_domain = I915_GEM_DOMAIN_CPU;
416 	if (cpu_write_needs_clflush(obj))
417 		obj->cache_dirty = true;
418 }
419 
420 int i915_gem_object_wait(struct drm_i915_gem_object *obj,
421 			 unsigned int flags,
422 			 long timeout);
423 int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
424 				  unsigned int flags,
425 				  const struct i915_sched_attr *attr);
426 #define I915_PRIORITY_DISPLAY I915_USER_PRIORITY(I915_PRIORITY_MAX)
427 
428 #endif
429