1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2016 Intel Corporation 5 */ 6 7 #include <linux/dma-fence-array.h> 8 #include <linux/dma-fence-chain.h> 9 #include <linux/jiffies.h> 10 11 #include "gt/intel_engine.h" 12 #include "gt/intel_rps.h" 13 14 #include "i915_gem_ioctls.h" 15 #include "i915_gem_object.h" 16 17 static long 18 i915_gem_object_wait_fence(struct dma_fence *fence, 19 unsigned int flags, 20 long timeout) 21 { 22 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1); 23 24 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 25 return timeout; 26 27 if (dma_fence_is_i915(fence)) 28 return i915_request_wait_timeout(to_request(fence), flags, timeout); 29 30 return dma_fence_wait_timeout(fence, 31 flags & I915_WAIT_INTERRUPTIBLE, 32 timeout); 33 } 34 35 static void 36 i915_gem_object_boost(struct dma_resv *resv, unsigned int flags) 37 { 38 struct dma_resv_iter cursor; 39 struct dma_fence *fence; 40 41 /* 42 * Prescan all fences for potential boosting before we begin waiting. 43 * 44 * When we wait, we wait on outstanding fences serially. If the 45 * dma-resv contains a sequence such as 1:1, 1:2 instead of a reduced 46 * form 1:2, then as we look at each wait in turn we see that each 47 * request is currently executing and not worthy of boosting. But if 48 * we only happen to look at the final fence in the sequence (because 49 * of request coalescing or splitting between read/write arrays by 50 * the iterator), then we would boost. As such our decision to boost 51 * or not is delicately balanced on the order we wait on fences. 52 * 53 * So instead of looking for boosts sequentially, look for all boosts 54 * upfront and then wait on the outstanding fences. 55 */ 56 57 dma_resv_iter_begin(&cursor, resv, 58 dma_resv_usage_rw(flags & I915_WAIT_ALL)); 59 dma_resv_for_each_fence_unlocked(&cursor, fence) 60 if (dma_fence_is_i915(fence) && 61 !i915_request_started(to_request(fence))) 62 intel_rps_boost(to_request(fence)); 63 dma_resv_iter_end(&cursor); 64 } 65 66 static long 67 i915_gem_object_wait_reservation(struct dma_resv *resv, 68 unsigned int flags, 69 long timeout) 70 { 71 struct dma_resv_iter cursor; 72 struct dma_fence *fence; 73 long ret = timeout ?: 1; 74 75 i915_gem_object_boost(resv, flags); 76 77 dma_resv_iter_begin(&cursor, resv, 78 dma_resv_usage_rw(flags & I915_WAIT_ALL)); 79 dma_resv_for_each_fence_unlocked(&cursor, fence) { 80 ret = i915_gem_object_wait_fence(fence, flags, timeout); 81 if (ret <= 0) 82 break; 83 84 if (timeout) 85 timeout = ret; 86 } 87 dma_resv_iter_end(&cursor); 88 89 return ret; 90 } 91 92 static void fence_set_priority(struct dma_fence *fence, 93 const struct i915_sched_attr *attr) 94 { 95 struct i915_request *rq; 96 struct intel_engine_cs *engine; 97 98 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence)) 99 return; 100 101 rq = to_request(fence); 102 engine = rq->engine; 103 104 rcu_read_lock(); /* RCU serialisation for set-wedged protection */ 105 if (engine->sched_engine->schedule) 106 engine->sched_engine->schedule(rq, attr); 107 rcu_read_unlock(); 108 } 109 110 static inline bool __dma_fence_is_chain(const struct dma_fence *fence) 111 { 112 return fence->ops == &dma_fence_chain_ops; 113 } 114 115 void i915_gem_fence_wait_priority(struct dma_fence *fence, 116 const struct i915_sched_attr *attr) 117 { 118 if (dma_fence_is_signaled(fence)) 119 return; 120 121 local_bh_disable(); 122 123 /* Recurse once into a fence-array */ 124 if (dma_fence_is_array(fence)) { 125 struct dma_fence_array *array = to_dma_fence_array(fence); 126 int i; 127 128 for (i = 0; i < array->num_fences; i++) 129 fence_set_priority(array->fences[i], attr); 130 } else if (__dma_fence_is_chain(fence)) { 131 struct dma_fence *iter; 132 133 /* The chain is ordered; if we boost the last, we boost all */ 134 dma_fence_chain_for_each(iter, fence) { 135 fence_set_priority(to_dma_fence_chain(iter)->fence, 136 attr); 137 break; 138 } 139 dma_fence_put(iter); 140 } else { 141 fence_set_priority(fence, attr); 142 } 143 144 local_bh_enable(); /* kick the tasklets if queues were reprioritised */ 145 } 146 147 int 148 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, 149 unsigned int flags, 150 const struct i915_sched_attr *attr) 151 { 152 struct dma_resv_iter cursor; 153 struct dma_fence *fence; 154 155 dma_resv_iter_begin(&cursor, obj->base.resv, 156 dma_resv_usage_rw(flags & I915_WAIT_ALL)); 157 dma_resv_for_each_fence_unlocked(&cursor, fence) 158 i915_gem_fence_wait_priority(fence, attr); 159 dma_resv_iter_end(&cursor); 160 return 0; 161 } 162 163 /** 164 * i915_gem_object_wait - Waits for rendering to the object to be completed 165 * @obj: i915 gem object 166 * @flags: how to wait (under a lock, for all rendering or just for writes etc) 167 * @timeout: how long to wait 168 */ 169 int 170 i915_gem_object_wait(struct drm_i915_gem_object *obj, 171 unsigned int flags, 172 long timeout) 173 { 174 might_sleep(); 175 GEM_BUG_ON(timeout < 0); 176 177 timeout = i915_gem_object_wait_reservation(obj->base.resv, 178 flags, timeout); 179 180 if (timeout < 0) 181 return timeout; 182 183 return !timeout ? -ETIME : 0; 184 } 185 186 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n) 187 { 188 /* nsecs_to_jiffies64() does not guard against overflow */ 189 if ((NSEC_PER_SEC % HZ) != 0 && 190 div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ) 191 return MAX_JIFFY_OFFSET; 192 193 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1); 194 } 195 196 static unsigned long to_wait_timeout(s64 timeout_ns) 197 { 198 if (timeout_ns < 0) 199 return MAX_SCHEDULE_TIMEOUT; 200 201 if (timeout_ns == 0) 202 return 0; 203 204 return nsecs_to_jiffies_timeout(timeout_ns); 205 } 206 207 /** 208 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT 209 * @dev: drm device pointer 210 * @data: ioctl data blob 211 * @file: drm file pointer 212 * 213 * Returns 0 if successful, else an error is returned with the remaining time in 214 * the timeout parameter. 215 * -ETIME: object is still busy after timeout 216 * -ERESTARTSYS: signal interrupted the wait 217 * -ENONENT: object doesn't exist 218 * Also possible, but rare: 219 * -EAGAIN: incomplete, restart syscall 220 * -ENOMEM: damn 221 * -ENODEV: Internal IRQ fail 222 * -E?: The add request failed 223 * 224 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any 225 * non-zero timeout parameter the wait ioctl will wait for the given number of 226 * nanoseconds on an object becoming unbusy. Since the wait itself does so 227 * without holding struct_mutex the object may become re-busied before this 228 * function completes. A similar but shorter * race condition exists in the busy 229 * ioctl 230 */ 231 int 232 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 233 { 234 struct drm_i915_gem_wait *args = data; 235 struct drm_i915_gem_object *obj; 236 ktime_t start; 237 long ret; 238 239 if (args->flags != 0) 240 return -EINVAL; 241 242 obj = i915_gem_object_lookup(file, args->bo_handle); 243 if (!obj) 244 return -ENOENT; 245 246 start = ktime_get(); 247 248 ret = i915_gem_object_wait(obj, 249 I915_WAIT_INTERRUPTIBLE | 250 I915_WAIT_PRIORITY | 251 I915_WAIT_ALL, 252 to_wait_timeout(args->timeout_ns)); 253 254 if (args->timeout_ns > 0) { 255 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start)); 256 if (args->timeout_ns < 0) 257 args->timeout_ns = 0; 258 259 /* 260 * Apparently ktime isn't accurate enough and occasionally has a 261 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch 262 * things up to make the test happy. We allow up to 1 jiffy. 263 * 264 * This is a regression from the timespec->ktime conversion. 265 */ 266 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns)) 267 args->timeout_ns = 0; 268 269 /* Asked to wait beyond the jiffie/scheduler precision? */ 270 if (ret == -ETIME && args->timeout_ns) 271 ret = -EAGAIN; 272 } 273 274 i915_gem_object_put(obj); 275 return ret; 276 } 277 278 /** 279 * i915_gem_object_wait_migration - Sync an accelerated migration operation 280 * @obj: The migrating object. 281 * @flags: waiting flags. Currently supports only I915_WAIT_INTERRUPTIBLE. 282 * 283 * Wait for any pending async migration operation on the object, 284 * whether it's explicitly (i915_gem_object_migrate()) or implicitly 285 * (swapin, initial clearing) initiated. 286 * 287 * Return: 0 if successful, -ERESTARTSYS if a signal was hit during waiting. 288 */ 289 int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj, 290 unsigned int flags) 291 { 292 might_sleep(); 293 294 return i915_gem_object_wait_moving_fence(obj, !!(flags & I915_WAIT_INTERRUPTIBLE)); 295 } 296