1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2016 Intel Corporation 5 */ 6 7 #include <linux/dma-fence-array.h> 8 #include <linux/jiffies.h> 9 10 #include "gt/intel_engine.h" 11 12 #include "i915_gem_ioctls.h" 13 #include "i915_gem_object.h" 14 15 static long 16 i915_gem_object_wait_fence(struct dma_fence *fence, 17 unsigned int flags, 18 long timeout) 19 { 20 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1); 21 22 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 23 return timeout; 24 25 if (dma_fence_is_i915(fence)) 26 return i915_request_wait(to_request(fence), flags, timeout); 27 28 return dma_fence_wait_timeout(fence, 29 flags & I915_WAIT_INTERRUPTIBLE, 30 timeout); 31 } 32 33 static long 34 i915_gem_object_wait_reservation(struct dma_resv *resv, 35 unsigned int flags, 36 long timeout) 37 { 38 struct dma_fence *excl; 39 bool prune_fences = false; 40 41 if (flags & I915_WAIT_ALL) { 42 struct dma_fence **shared; 43 unsigned int count, i; 44 int ret; 45 46 ret = dma_resv_get_fences_rcu(resv, 47 &excl, &count, &shared); 48 if (ret) 49 return ret; 50 51 for (i = 0; i < count; i++) { 52 timeout = i915_gem_object_wait_fence(shared[i], 53 flags, timeout); 54 if (timeout < 0) 55 break; 56 57 dma_fence_put(shared[i]); 58 } 59 60 for (; i < count; i++) 61 dma_fence_put(shared[i]); 62 kfree(shared); 63 64 /* 65 * If both shared fences and an exclusive fence exist, 66 * then by construction the shared fences must be later 67 * than the exclusive fence. If we successfully wait for 68 * all the shared fences, we know that the exclusive fence 69 * must all be signaled. If all the shared fences are 70 * signaled, we can prune the array and recover the 71 * floating references on the fences/requests. 72 */ 73 prune_fences = count && timeout >= 0; 74 } else { 75 excl = dma_resv_get_excl_rcu(resv); 76 } 77 78 if (excl && timeout >= 0) 79 timeout = i915_gem_object_wait_fence(excl, flags, timeout); 80 81 dma_fence_put(excl); 82 83 /* 84 * Opportunistically prune the fences iff we know they have *all* been 85 * signaled. 86 */ 87 if (prune_fences && dma_resv_trylock(resv)) { 88 if (dma_resv_test_signaled_rcu(resv, true)) 89 dma_resv_add_excl_fence(resv, NULL); 90 dma_resv_unlock(resv); 91 } 92 93 return timeout; 94 } 95 96 static void __fence_set_priority(struct dma_fence *fence, 97 const struct i915_sched_attr *attr) 98 { 99 struct i915_request *rq; 100 struct intel_engine_cs *engine; 101 102 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence)) 103 return; 104 105 rq = to_request(fence); 106 engine = rq->engine; 107 108 local_bh_disable(); 109 rcu_read_lock(); /* RCU serialisation for set-wedged protection */ 110 if (engine->schedule) 111 engine->schedule(rq, attr); 112 rcu_read_unlock(); 113 local_bh_enable(); /* kick the tasklets if queues were reprioritised */ 114 } 115 116 static void fence_set_priority(struct dma_fence *fence, 117 const struct i915_sched_attr *attr) 118 { 119 /* Recurse once into a fence-array */ 120 if (dma_fence_is_array(fence)) { 121 struct dma_fence_array *array = to_dma_fence_array(fence); 122 int i; 123 124 for (i = 0; i < array->num_fences; i++) 125 __fence_set_priority(array->fences[i], attr); 126 } else { 127 __fence_set_priority(fence, attr); 128 } 129 } 130 131 int 132 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 133 unsigned int flags, 134 const struct i915_sched_attr *attr) 135 { 136 struct dma_fence *excl; 137 138 if (flags & I915_WAIT_ALL) { 139 struct dma_fence **shared; 140 unsigned int count, i; 141 int ret; 142 143 ret = dma_resv_get_fences_rcu(obj->base.resv, 144 &excl, &count, &shared); 145 if (ret) 146 return ret; 147 148 for (i = 0; i < count; i++) { 149 fence_set_priority(shared[i], attr); 150 dma_fence_put(shared[i]); 151 } 152 153 kfree(shared); 154 } else { 155 excl = dma_resv_get_excl_rcu(obj->base.resv); 156 } 157 158 if (excl) { 159 fence_set_priority(excl, attr); 160 dma_fence_put(excl); 161 } 162 return 0; 163 } 164 165 /** 166 * Waits for rendering to the object to be completed 167 * @obj: i915 gem object 168 * @flags: how to wait (under a lock, for all rendering or just for writes etc) 169 * @timeout: how long to wait 170 */ 171 int 172 i915_gem_object_wait(struct drm_i915_gem_object *obj, 173 unsigned int flags, 174 long timeout) 175 { 176 might_sleep(); 177 GEM_BUG_ON(timeout < 0); 178 179 timeout = i915_gem_object_wait_reservation(obj->base.resv, 180 flags, timeout); 181 return timeout < 0 ? timeout : 0; 182 } 183 184 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n) 185 { 186 /* nsecs_to_jiffies64() does not guard against overflow */ 187 if (NSEC_PER_SEC % HZ && 188 div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ) 189 return MAX_JIFFY_OFFSET; 190 191 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1); 192 } 193 194 static unsigned long to_wait_timeout(s64 timeout_ns) 195 { 196 if (timeout_ns < 0) 197 return MAX_SCHEDULE_TIMEOUT; 198 199 if (timeout_ns == 0) 200 return 0; 201 202 return nsecs_to_jiffies_timeout(timeout_ns); 203 } 204 205 /** 206 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT 207 * @dev: drm device pointer 208 * @data: ioctl data blob 209 * @file: drm file pointer 210 * 211 * Returns 0 if successful, else an error is returned with the remaining time in 212 * the timeout parameter. 213 * -ETIME: object is still busy after timeout 214 * -ERESTARTSYS: signal interrupted the wait 215 * -ENONENT: object doesn't exist 216 * Also possible, but rare: 217 * -EAGAIN: incomplete, restart syscall 218 * -ENOMEM: damn 219 * -ENODEV: Internal IRQ fail 220 * -E?: The add request failed 221 * 222 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any 223 * non-zero timeout parameter the wait ioctl will wait for the given number of 224 * nanoseconds on an object becoming unbusy. Since the wait itself does so 225 * without holding struct_mutex the object may become re-busied before this 226 * function completes. A similar but shorter * race condition exists in the busy 227 * ioctl 228 */ 229 int 230 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 231 { 232 struct drm_i915_gem_wait *args = data; 233 struct drm_i915_gem_object *obj; 234 ktime_t start; 235 long ret; 236 237 if (args->flags != 0) 238 return -EINVAL; 239 240 obj = i915_gem_object_lookup(file, args->bo_handle); 241 if (!obj) 242 return -ENOENT; 243 244 start = ktime_get(); 245 246 ret = i915_gem_object_wait(obj, 247 I915_WAIT_INTERRUPTIBLE | 248 I915_WAIT_PRIORITY | 249 I915_WAIT_ALL, 250 to_wait_timeout(args->timeout_ns)); 251 252 if (args->timeout_ns > 0) { 253 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start)); 254 if (args->timeout_ns < 0) 255 args->timeout_ns = 0; 256 257 /* 258 * Apparently ktime isn't accurate enough and occasionally has a 259 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch 260 * things up to make the test happy. We allow up to 1 jiffy. 261 * 262 * This is a regression from the timespec->ktime conversion. 263 */ 264 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns)) 265 args->timeout_ns = 0; 266 267 /* Asked to wait beyond the jiffie/scheduler precision? */ 268 if (ret == -ETIME && args->timeout_ns) 269 ret = -EAGAIN; 270 } 271 272 i915_gem_object_put(obj); 273 return ret; 274 } 275