1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2016 Intel Corporation 5 */ 6 7 #include <linux/dma-fence-array.h> 8 #include <linux/jiffies.h> 9 10 #include "gt/intel_engine.h" 11 12 #include "i915_gem_ioctls.h" 13 #include "i915_gem_object.h" 14 15 static long 16 i915_gem_object_wait_fence(struct dma_fence *fence, 17 unsigned int flags, 18 long timeout) 19 { 20 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1); 21 22 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 23 return timeout; 24 25 if (dma_fence_is_i915(fence)) 26 return i915_request_wait(to_request(fence), flags, timeout); 27 28 return dma_fence_wait_timeout(fence, 29 flags & I915_WAIT_INTERRUPTIBLE, 30 timeout); 31 } 32 33 static long 34 i915_gem_object_wait_reservation(struct reservation_object *resv, 35 unsigned int flags, 36 long timeout) 37 { 38 unsigned int seq = __read_seqcount_begin(&resv->seq); 39 struct dma_fence *excl; 40 bool prune_fences = false; 41 42 if (flags & I915_WAIT_ALL) { 43 struct dma_fence **shared; 44 unsigned int count, i; 45 int ret; 46 47 ret = reservation_object_get_fences_rcu(resv, 48 &excl, &count, &shared); 49 if (ret) 50 return ret; 51 52 for (i = 0; i < count; i++) { 53 timeout = i915_gem_object_wait_fence(shared[i], 54 flags, timeout); 55 if (timeout < 0) 56 break; 57 58 dma_fence_put(shared[i]); 59 } 60 61 for (; i < count; i++) 62 dma_fence_put(shared[i]); 63 kfree(shared); 64 65 /* 66 * If both shared fences and an exclusive fence exist, 67 * then by construction the shared fences must be later 68 * than the exclusive fence. If we successfully wait for 69 * all the shared fences, we know that the exclusive fence 70 * must all be signaled. If all the shared fences are 71 * signaled, we can prune the array and recover the 72 * floating references on the fences/requests. 73 */ 74 prune_fences = count && timeout >= 0; 75 } else { 76 excl = reservation_object_get_excl_rcu(resv); 77 } 78 79 if (excl && timeout >= 0) 80 timeout = i915_gem_object_wait_fence(excl, flags, timeout); 81 82 dma_fence_put(excl); 83 84 /* 85 * Opportunistically prune the fences iff we know they have *all* been 86 * signaled and that the reservation object has not been changed (i.e. 87 * no new fences have been added). 88 */ 89 if (prune_fences && !__read_seqcount_retry(&resv->seq, seq)) { 90 if (reservation_object_trylock(resv)) { 91 if (!__read_seqcount_retry(&resv->seq, seq)) 92 reservation_object_add_excl_fence(resv, NULL); 93 reservation_object_unlock(resv); 94 } 95 } 96 97 return timeout; 98 } 99 100 static void __fence_set_priority(struct dma_fence *fence, 101 const struct i915_sched_attr *attr) 102 { 103 struct i915_request *rq; 104 struct intel_engine_cs *engine; 105 106 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence)) 107 return; 108 109 rq = to_request(fence); 110 engine = rq->engine; 111 112 local_bh_disable(); 113 rcu_read_lock(); /* RCU serialisation for set-wedged protection */ 114 if (engine->schedule) 115 engine->schedule(rq, attr); 116 rcu_read_unlock(); 117 local_bh_enable(); /* kick the tasklets if queues were reprioritised */ 118 } 119 120 static void fence_set_priority(struct dma_fence *fence, 121 const struct i915_sched_attr *attr) 122 { 123 /* Recurse once into a fence-array */ 124 if (dma_fence_is_array(fence)) { 125 struct dma_fence_array *array = to_dma_fence_array(fence); 126 int i; 127 128 for (i = 0; i < array->num_fences; i++) 129 __fence_set_priority(array->fences[i], attr); 130 } else { 131 __fence_set_priority(fence, attr); 132 } 133 } 134 135 int 136 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 137 unsigned int flags, 138 const struct i915_sched_attr *attr) 139 { 140 struct dma_fence *excl; 141 142 if (flags & I915_WAIT_ALL) { 143 struct dma_fence **shared; 144 unsigned int count, i; 145 int ret; 146 147 ret = reservation_object_get_fences_rcu(obj->base.resv, 148 &excl, &count, &shared); 149 if (ret) 150 return ret; 151 152 for (i = 0; i < count; i++) { 153 fence_set_priority(shared[i], attr); 154 dma_fence_put(shared[i]); 155 } 156 157 kfree(shared); 158 } else { 159 excl = reservation_object_get_excl_rcu(obj->base.resv); 160 } 161 162 if (excl) { 163 fence_set_priority(excl, attr); 164 dma_fence_put(excl); 165 } 166 return 0; 167 } 168 169 /** 170 * Waits for rendering to the object to be completed 171 * @obj: i915 gem object 172 * @flags: how to wait (under a lock, for all rendering or just for writes etc) 173 * @timeout: how long to wait 174 */ 175 int 176 i915_gem_object_wait(struct drm_i915_gem_object *obj, 177 unsigned int flags, 178 long timeout) 179 { 180 might_sleep(); 181 GEM_BUG_ON(timeout < 0); 182 183 timeout = i915_gem_object_wait_reservation(obj->base.resv, 184 flags, timeout); 185 return timeout < 0 ? timeout : 0; 186 } 187 188 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n) 189 { 190 /* nsecs_to_jiffies64() does not guard against overflow */ 191 if (NSEC_PER_SEC % HZ && 192 div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ) 193 return MAX_JIFFY_OFFSET; 194 195 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1); 196 } 197 198 static unsigned long to_wait_timeout(s64 timeout_ns) 199 { 200 if (timeout_ns < 0) 201 return MAX_SCHEDULE_TIMEOUT; 202 203 if (timeout_ns == 0) 204 return 0; 205 206 return nsecs_to_jiffies_timeout(timeout_ns); 207 } 208 209 /** 210 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT 211 * @dev: drm device pointer 212 * @data: ioctl data blob 213 * @file: drm file pointer 214 * 215 * Returns 0 if successful, else an error is returned with the remaining time in 216 * the timeout parameter. 217 * -ETIME: object is still busy after timeout 218 * -ERESTARTSYS: signal interrupted the wait 219 * -ENONENT: object doesn't exist 220 * Also possible, but rare: 221 * -EAGAIN: incomplete, restart syscall 222 * -ENOMEM: damn 223 * -ENODEV: Internal IRQ fail 224 * -E?: The add request failed 225 * 226 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any 227 * non-zero timeout parameter the wait ioctl will wait for the given number of 228 * nanoseconds on an object becoming unbusy. Since the wait itself does so 229 * without holding struct_mutex the object may become re-busied before this 230 * function completes. A similar but shorter * race condition exists in the busy 231 * ioctl 232 */ 233 int 234 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 235 { 236 struct drm_i915_gem_wait *args = data; 237 struct drm_i915_gem_object *obj; 238 ktime_t start; 239 long ret; 240 241 if (args->flags != 0) 242 return -EINVAL; 243 244 obj = i915_gem_object_lookup(file, args->bo_handle); 245 if (!obj) 246 return -ENOENT; 247 248 start = ktime_get(); 249 250 ret = i915_gem_object_wait(obj, 251 I915_WAIT_INTERRUPTIBLE | 252 I915_WAIT_PRIORITY | 253 I915_WAIT_ALL, 254 to_wait_timeout(args->timeout_ns)); 255 256 if (args->timeout_ns > 0) { 257 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start)); 258 if (args->timeout_ns < 0) 259 args->timeout_ns = 0; 260 261 /* 262 * Apparently ktime isn't accurate enough and occasionally has a 263 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch 264 * things up to make the test happy. We allow up to 1 jiffy. 265 * 266 * This is a regression from the timespec->ktime conversion. 267 */ 268 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns)) 269 args->timeout_ns = 0; 270 271 /* Asked to wait beyond the jiffie/scheduler precision? */ 272 if (ret == -ETIME && args->timeout_ns) 273 ret = -EAGAIN; 274 } 275 276 i915_gem_object_put(obj); 277 return ret; 278 } 279