1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2
3 #include <drm/drm_exec.h>
4 #include <drm/drm_gem.h>
5 #include <linux/dma-resv.h>
6
7 /**
8 * DOC: Overview
9 *
10 * This component mainly abstracts the retry loop necessary for locking
11 * multiple GEM objects while preparing hardware operations (e.g. command
12 * submissions, page table updates etc..).
13 *
14 * If a contention is detected while locking a GEM object the cleanup procedure
15 * unlocks all previously locked GEM objects and locks the contended one first
16 * before locking any further objects.
17 *
18 * After an object is locked fences slots can optionally be reserved on the
19 * dma_resv object inside the GEM object.
20 *
21 * A typical usage pattern should look like this::
22 *
23 * struct drm_gem_object *obj;
24 * struct drm_exec exec;
25 * unsigned long index;
26 * int ret;
27 *
28 * drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
29 * drm_exec_until_all_locked(&exec) {
30 * ret = drm_exec_prepare_obj(&exec, boA, 1);
31 * drm_exec_retry_on_contention(&exec);
32 * if (ret)
33 * goto error;
34 *
35 * ret = drm_exec_prepare_obj(&exec, boB, 1);
36 * drm_exec_retry_on_contention(&exec);
37 * if (ret)
38 * goto error;
39 * }
40 *
41 * drm_exec_for_each_locked_object(&exec, index, obj) {
42 * dma_resv_add_fence(obj->resv, fence, DMA_RESV_USAGE_READ);
43 * ...
44 * }
45 * drm_exec_fini(&exec);
46 *
47 * See struct dma_exec for more details.
48 */
49
50 /* Dummy value used to initially enter the retry loop */
51 #define DRM_EXEC_DUMMY ((void *)~0)
52
53 /* Unlock all objects and drop references */
drm_exec_unlock_all(struct drm_exec * exec)54 static void drm_exec_unlock_all(struct drm_exec *exec)
55 {
56 struct drm_gem_object *obj;
57 unsigned long index;
58
59 drm_exec_for_each_locked_object_reverse(exec, index, obj) {
60 dma_resv_unlock(obj->resv);
61 drm_gem_object_put(obj);
62 }
63
64 drm_gem_object_put(exec->prelocked);
65 exec->prelocked = NULL;
66 }
67
68 /**
69 * drm_exec_init - initialize a drm_exec object
70 * @exec: the drm_exec object to initialize
71 * @flags: controls locking behavior, see DRM_EXEC_* defines
72 *
73 * Initialize the object and make sure that we can track locked objects.
74 */
drm_exec_init(struct drm_exec * exec,uint32_t flags)75 void drm_exec_init(struct drm_exec *exec, uint32_t flags)
76 {
77 exec->flags = flags;
78 exec->objects = kmalloc(PAGE_SIZE, GFP_KERNEL);
79
80 /* If allocation here fails, just delay that till the first use */
81 exec->max_objects = exec->objects ? PAGE_SIZE / sizeof(void *) : 0;
82 exec->num_objects = 0;
83 exec->contended = DRM_EXEC_DUMMY;
84 exec->prelocked = NULL;
85 }
86 EXPORT_SYMBOL(drm_exec_init);
87
88 /**
89 * drm_exec_fini - finalize a drm_exec object
90 * @exec: the drm_exec object to finalize
91 *
92 * Unlock all locked objects, drop the references to objects and free all memory
93 * used for tracking the state.
94 */
drm_exec_fini(struct drm_exec * exec)95 void drm_exec_fini(struct drm_exec *exec)
96 {
97 drm_exec_unlock_all(exec);
98 kvfree(exec->objects);
99 if (exec->contended != DRM_EXEC_DUMMY) {
100 drm_gem_object_put(exec->contended);
101 ww_acquire_fini(&exec->ticket);
102 }
103 }
104 EXPORT_SYMBOL(drm_exec_fini);
105
106 /**
107 * drm_exec_cleanup - cleanup when contention is detected
108 * @exec: the drm_exec object to cleanup
109 *
110 * Cleanup the current state and return true if we should stay inside the retry
111 * loop, false if there wasn't any contention detected and we can keep the
112 * objects locked.
113 */
drm_exec_cleanup(struct drm_exec * exec)114 bool drm_exec_cleanup(struct drm_exec *exec)
115 {
116 if (likely(!exec->contended)) {
117 ww_acquire_done(&exec->ticket);
118 return false;
119 }
120
121 if (likely(exec->contended == DRM_EXEC_DUMMY)) {
122 exec->contended = NULL;
123 ww_acquire_init(&exec->ticket, &reservation_ww_class);
124 return true;
125 }
126
127 drm_exec_unlock_all(exec);
128 exec->num_objects = 0;
129 return true;
130 }
131 EXPORT_SYMBOL(drm_exec_cleanup);
132
133 /* Track the locked object in the array */
drm_exec_obj_locked(struct drm_exec * exec,struct drm_gem_object * obj)134 static int drm_exec_obj_locked(struct drm_exec *exec,
135 struct drm_gem_object *obj)
136 {
137 if (unlikely(exec->num_objects == exec->max_objects)) {
138 size_t size = exec->max_objects * sizeof(void *);
139 void *tmp;
140
141 tmp = kvrealloc(exec->objects, size, size + PAGE_SIZE,
142 GFP_KERNEL);
143 if (!tmp)
144 return -ENOMEM;
145
146 exec->objects = tmp;
147 exec->max_objects += PAGE_SIZE / sizeof(void *);
148 }
149 drm_gem_object_get(obj);
150 exec->objects[exec->num_objects++] = obj;
151
152 return 0;
153 }
154
155 /* Make sure the contended object is locked first */
drm_exec_lock_contended(struct drm_exec * exec)156 static int drm_exec_lock_contended(struct drm_exec *exec)
157 {
158 struct drm_gem_object *obj = exec->contended;
159 int ret;
160
161 if (likely(!obj))
162 return 0;
163
164 /* Always cleanup the contention so that error handling can kick in */
165 exec->contended = NULL;
166 if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT) {
167 ret = dma_resv_lock_slow_interruptible(obj->resv,
168 &exec->ticket);
169 if (unlikely(ret))
170 goto error_dropref;
171 } else {
172 dma_resv_lock_slow(obj->resv, &exec->ticket);
173 }
174
175 ret = drm_exec_obj_locked(exec, obj);
176 if (unlikely(ret))
177 goto error_unlock;
178
179 exec->prelocked = obj;
180 return 0;
181
182 error_unlock:
183 dma_resv_unlock(obj->resv);
184
185 error_dropref:
186 drm_gem_object_put(obj);
187 return ret;
188 }
189
190 /**
191 * drm_exec_lock_obj - lock a GEM object for use
192 * @exec: the drm_exec object with the state
193 * @obj: the GEM object to lock
194 *
195 * Lock a GEM object for use and grab a reference to it.
196 *
197 * Returns: -EDEADLK if a contention is detected, -EALREADY when object is
198 * already locked (can be suppressed by setting the DRM_EXEC_IGNORE_DUPLICATES
199 * flag), -ENOMEM when memory allocation failed and zero for success.
200 */
drm_exec_lock_obj(struct drm_exec * exec,struct drm_gem_object * obj)201 int drm_exec_lock_obj(struct drm_exec *exec, struct drm_gem_object *obj)
202 {
203 int ret;
204
205 ret = drm_exec_lock_contended(exec);
206 if (unlikely(ret))
207 return ret;
208
209 if (exec->prelocked == obj) {
210 drm_gem_object_put(exec->prelocked);
211 exec->prelocked = NULL;
212 return 0;
213 }
214
215 if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT)
216 ret = dma_resv_lock_interruptible(obj->resv, &exec->ticket);
217 else
218 ret = dma_resv_lock(obj->resv, &exec->ticket);
219
220 if (unlikely(ret == -EDEADLK)) {
221 drm_gem_object_get(obj);
222 exec->contended = obj;
223 return -EDEADLK;
224 }
225
226 if (unlikely(ret == -EALREADY) &&
227 exec->flags & DRM_EXEC_IGNORE_DUPLICATES)
228 return 0;
229
230 if (unlikely(ret))
231 return ret;
232
233 ret = drm_exec_obj_locked(exec, obj);
234 if (ret)
235 goto error_unlock;
236
237 return 0;
238
239 error_unlock:
240 dma_resv_unlock(obj->resv);
241 return ret;
242 }
243 EXPORT_SYMBOL(drm_exec_lock_obj);
244
245 /**
246 * drm_exec_unlock_obj - unlock a GEM object in this exec context
247 * @exec: the drm_exec object with the state
248 * @obj: the GEM object to unlock
249 *
250 * Unlock the GEM object and remove it from the collection of locked objects.
251 * Should only be used to unlock the most recently locked objects. It's not time
252 * efficient to unlock objects locked long ago.
253 */
drm_exec_unlock_obj(struct drm_exec * exec,struct drm_gem_object * obj)254 void drm_exec_unlock_obj(struct drm_exec *exec, struct drm_gem_object *obj)
255 {
256 unsigned int i;
257
258 for (i = exec->num_objects; i--;) {
259 if (exec->objects[i] == obj) {
260 dma_resv_unlock(obj->resv);
261 for (++i; i < exec->num_objects; ++i)
262 exec->objects[i - 1] = exec->objects[i];
263 --exec->num_objects;
264 drm_gem_object_put(obj);
265 return;
266 }
267
268 }
269 }
270 EXPORT_SYMBOL(drm_exec_unlock_obj);
271
272 /**
273 * drm_exec_prepare_obj - prepare a GEM object for use
274 * @exec: the drm_exec object with the state
275 * @obj: the GEM object to prepare
276 * @num_fences: how many fences to reserve
277 *
278 * Prepare a GEM object for use by locking it and reserving fence slots.
279 *
280 * Returns: -EDEADLK if a contention is detected, -EALREADY when object is
281 * already locked, -ENOMEM when memory allocation failed and zero for success.
282 */
drm_exec_prepare_obj(struct drm_exec * exec,struct drm_gem_object * obj,unsigned int num_fences)283 int drm_exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj,
284 unsigned int num_fences)
285 {
286 int ret;
287
288 ret = drm_exec_lock_obj(exec, obj);
289 if (ret)
290 return ret;
291
292 ret = dma_resv_reserve_fences(obj->resv, num_fences);
293 if (ret) {
294 drm_exec_unlock_obj(exec, obj);
295 return ret;
296 }
297
298 return 0;
299 }
300 EXPORT_SYMBOL(drm_exec_prepare_obj);
301
302 /**
303 * drm_exec_prepare_array - helper to prepare an array of objects
304 * @exec: the drm_exec object with the state
305 * @objects: array of GEM object to prepare
306 * @num_objects: number of GEM objects in the array
307 * @num_fences: number of fences to reserve on each GEM object
308 *
309 * Prepares all GEM objects in an array, aborts on first error.
310 * Reserves @num_fences on each GEM object after locking it.
311 *
312 * Returns: -EDEADLOCK on contention, -EALREADY when object is already locked,
313 * -ENOMEM when memory allocation failed and zero for success.
314 */
drm_exec_prepare_array(struct drm_exec * exec,struct drm_gem_object ** objects,unsigned int num_objects,unsigned int num_fences)315 int drm_exec_prepare_array(struct drm_exec *exec,
316 struct drm_gem_object **objects,
317 unsigned int num_objects,
318 unsigned int num_fences)
319 {
320 int ret;
321
322 for (unsigned int i = 0; i < num_objects; ++i) {
323 ret = drm_exec_prepare_obj(exec, objects[i], num_fences);
324 if (unlikely(ret))
325 return ret;
326 }
327
328 return 0;
329 }
330 EXPORT_SYMBOL(drm_exec_prepare_array);
331
332 MODULE_DESCRIPTION("DRM execution context");
333 MODULE_LICENSE("Dual MIT/GPL");
334