1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2016 Intel Corporation
5  */
6 
7 #include <linux/dma-fence-array.h>
8 #include <linux/dma-fence-chain.h>
9 #include <linux/jiffies.h>
10 
11 #include "gt/intel_engine.h"
12 
13 #include "dma_resv_utils.h"
14 #include "i915_gem_ioctls.h"
15 #include "i915_gem_object.h"
16 
17 static long
18 i915_gem_object_wait_fence(struct dma_fence *fence,
19 			   unsigned int flags,
20 			   long timeout)
21 {
22 	BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
23 
24 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
25 		return timeout;
26 
27 	if (dma_fence_is_i915(fence))
28 		return i915_request_wait(to_request(fence), flags, timeout);
29 
30 	return dma_fence_wait_timeout(fence,
31 				      flags & I915_WAIT_INTERRUPTIBLE,
32 				      timeout);
33 }
34 
35 static long
36 i915_gem_object_wait_reservation(struct dma_resv *resv,
37 				 unsigned int flags,
38 				 long timeout)
39 {
40 	struct dma_fence *excl;
41 	bool prune_fences = false;
42 
43 	if (flags & I915_WAIT_ALL) {
44 		struct dma_fence **shared;
45 		unsigned int count, i;
46 		int ret;
47 
48 		ret = dma_resv_get_fences_rcu(resv, &excl, &count, &shared);
49 		if (ret)
50 			return ret;
51 
52 		for (i = 0; i < count; i++) {
53 			timeout = i915_gem_object_wait_fence(shared[i],
54 							     flags, timeout);
55 			if (timeout < 0)
56 				break;
57 
58 			dma_fence_put(shared[i]);
59 		}
60 
61 		for (; i < count; i++)
62 			dma_fence_put(shared[i]);
63 		kfree(shared);
64 
65 		/*
66 		 * If both shared fences and an exclusive fence exist,
67 		 * then by construction the shared fences must be later
68 		 * than the exclusive fence. If we successfully wait for
69 		 * all the shared fences, we know that the exclusive fence
70 		 * must all be signaled. If all the shared fences are
71 		 * signaled, we can prune the array and recover the
72 		 * floating references on the fences/requests.
73 		 */
74 		prune_fences = count && timeout >= 0;
75 	} else {
76 		excl = dma_resv_get_excl_rcu(resv);
77 	}
78 
79 	if (excl && timeout >= 0)
80 		timeout = i915_gem_object_wait_fence(excl, flags, timeout);
81 
82 	dma_fence_put(excl);
83 
84 	/*
85 	 * Opportunistically prune the fences iff we know they have *all* been
86 	 * signaled.
87 	 */
88 	if (prune_fences)
89 		dma_resv_prune(resv);
90 
91 	return timeout;
92 }
93 
94 static void fence_set_priority(struct dma_fence *fence,
95 			       const struct i915_sched_attr *attr)
96 {
97 	struct i915_request *rq;
98 	struct intel_engine_cs *engine;
99 
100 	if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
101 		return;
102 
103 	rq = to_request(fence);
104 	engine = rq->engine;
105 
106 	rcu_read_lock(); /* RCU serialisation for set-wedged protection */
107 	if (engine->schedule)
108 		engine->schedule(rq, attr);
109 	rcu_read_unlock();
110 }
111 
112 static inline bool __dma_fence_is_chain(const struct dma_fence *fence)
113 {
114 	return fence->ops == &dma_fence_chain_ops;
115 }
116 
117 void i915_gem_fence_wait_priority(struct dma_fence *fence,
118 				  const struct i915_sched_attr *attr)
119 {
120 	if (dma_fence_is_signaled(fence))
121 		return;
122 
123 	local_bh_disable();
124 
125 	/* Recurse once into a fence-array */
126 	if (dma_fence_is_array(fence)) {
127 		struct dma_fence_array *array = to_dma_fence_array(fence);
128 		int i;
129 
130 		for (i = 0; i < array->num_fences; i++)
131 			fence_set_priority(array->fences[i], attr);
132 	} else if (__dma_fence_is_chain(fence)) {
133 		struct dma_fence *iter;
134 
135 		/* The chain is ordered; if we boost the last, we boost all */
136 		dma_fence_chain_for_each(iter, fence) {
137 			fence_set_priority(to_dma_fence_chain(iter)->fence,
138 					   attr);
139 			break;
140 		}
141 		dma_fence_put(iter);
142 	} else {
143 		fence_set_priority(fence, attr);
144 	}
145 
146 	local_bh_enable(); /* kick the tasklets if queues were reprioritised */
147 }
148 
149 int
150 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
151 			      unsigned int flags,
152 			      const struct i915_sched_attr *attr)
153 {
154 	struct dma_fence *excl;
155 
156 	if (flags & I915_WAIT_ALL) {
157 		struct dma_fence **shared;
158 		unsigned int count, i;
159 		int ret;
160 
161 		ret = dma_resv_get_fences_rcu(obj->base.resv,
162 					      &excl, &count, &shared);
163 		if (ret)
164 			return ret;
165 
166 		for (i = 0; i < count; i++) {
167 			i915_gem_fence_wait_priority(shared[i], attr);
168 			dma_fence_put(shared[i]);
169 		}
170 
171 		kfree(shared);
172 	} else {
173 		excl = dma_resv_get_excl_rcu(obj->base.resv);
174 	}
175 
176 	if (excl) {
177 		i915_gem_fence_wait_priority(excl, attr);
178 		dma_fence_put(excl);
179 	}
180 	return 0;
181 }
182 
183 /**
184  * Waits for rendering to the object to be completed
185  * @obj: i915 gem object
186  * @flags: how to wait (under a lock, for all rendering or just for writes etc)
187  * @timeout: how long to wait
188  */
189 int
190 i915_gem_object_wait(struct drm_i915_gem_object *obj,
191 		     unsigned int flags,
192 		     long timeout)
193 {
194 	might_sleep();
195 	GEM_BUG_ON(timeout < 0);
196 
197 	timeout = i915_gem_object_wait_reservation(obj->base.resv,
198 						   flags, timeout);
199 	return timeout < 0 ? timeout : 0;
200 }
201 
202 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
203 {
204 	/* nsecs_to_jiffies64() does not guard against overflow */
205 	if (NSEC_PER_SEC % HZ &&
206 	    div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
207 		return MAX_JIFFY_OFFSET;
208 
209 	return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
210 }
211 
212 static unsigned long to_wait_timeout(s64 timeout_ns)
213 {
214 	if (timeout_ns < 0)
215 		return MAX_SCHEDULE_TIMEOUT;
216 
217 	if (timeout_ns == 0)
218 		return 0;
219 
220 	return nsecs_to_jiffies_timeout(timeout_ns);
221 }
222 
223 /**
224  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
225  * @dev: drm device pointer
226  * @data: ioctl data blob
227  * @file: drm file pointer
228  *
229  * Returns 0 if successful, else an error is returned with the remaining time in
230  * the timeout parameter.
231  *  -ETIME: object is still busy after timeout
232  *  -ERESTARTSYS: signal interrupted the wait
233  *  -ENONENT: object doesn't exist
234  * Also possible, but rare:
235  *  -EAGAIN: incomplete, restart syscall
236  *  -ENOMEM: damn
237  *  -ENODEV: Internal IRQ fail
238  *  -E?: The add request failed
239  *
240  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
241  * non-zero timeout parameter the wait ioctl will wait for the given number of
242  * nanoseconds on an object becoming unbusy. Since the wait itself does so
243  * without holding struct_mutex the object may become re-busied before this
244  * function completes. A similar but shorter * race condition exists in the busy
245  * ioctl
246  */
247 int
248 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
249 {
250 	struct drm_i915_gem_wait *args = data;
251 	struct drm_i915_gem_object *obj;
252 	ktime_t start;
253 	long ret;
254 
255 	if (args->flags != 0)
256 		return -EINVAL;
257 
258 	obj = i915_gem_object_lookup(file, args->bo_handle);
259 	if (!obj)
260 		return -ENOENT;
261 
262 	start = ktime_get();
263 
264 	ret = i915_gem_object_wait(obj,
265 				   I915_WAIT_INTERRUPTIBLE |
266 				   I915_WAIT_PRIORITY |
267 				   I915_WAIT_ALL,
268 				   to_wait_timeout(args->timeout_ns));
269 
270 	if (args->timeout_ns > 0) {
271 		args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
272 		if (args->timeout_ns < 0)
273 			args->timeout_ns = 0;
274 
275 		/*
276 		 * Apparently ktime isn't accurate enough and occasionally has a
277 		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
278 		 * things up to make the test happy. We allow up to 1 jiffy.
279 		 *
280 		 * This is a regression from the timespec->ktime conversion.
281 		 */
282 		if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
283 			args->timeout_ns = 0;
284 
285 		/* Asked to wait beyond the jiffie/scheduler precision? */
286 		if (ret == -ETIME && args->timeout_ns)
287 			ret = -EAGAIN;
288 	}
289 
290 	i915_gem_object_put(obj);
291 	return ret;
292 }
293