1 /* 2 * Copyright © 2008-2018 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #ifndef I915_REQUEST_H 26 #define I915_REQUEST_H 27 28 #include <linux/dma-fence.h> 29 30 #include "i915_gem.h" 31 #include "i915_scheduler.h" 32 #include "i915_sw_fence.h" 33 34 #include <uapi/drm/i915_drm.h> 35 36 struct drm_file; 37 struct drm_i915_gem_object; 38 struct i915_request; 39 struct i915_timeline; 40 41 struct i915_capture_list { 42 struct i915_capture_list *next; 43 struct i915_vma *vma; 44 }; 45 46 enum { 47 /* 48 * I915_FENCE_FLAG_ACTIVE - this request is currently submitted to HW. 49 * 50 * Set by __i915_request_submit() on handing over to HW, and cleared 51 * by __i915_request_unsubmit() if we preempt this request. 52 * 53 * Finally cleared for consistency on retiring the request, when 54 * we know the HW is no longer running this request. 55 * 56 * See i915_request_is_active() 57 */ 58 I915_FENCE_FLAG_ACTIVE = DMA_FENCE_FLAG_USER_BITS, 59 60 /* 61 * I915_FENCE_FLAG_SIGNAL - this request is currently on signal_list 62 * 63 * Internal bookkeeping used by the breadcrumb code to track when 64 * a request is on the various signal_list. 65 */ 66 I915_FENCE_FLAG_SIGNAL, 67 }; 68 69 /** 70 * Request queue structure. 71 * 72 * The request queue allows us to note sequence numbers that have been emitted 73 * and may be associated with active buffers to be retired. 74 * 75 * By keeping this list, we can avoid having to do questionable sequence 76 * number comparisons on buffer last_read|write_seqno. It also allows an 77 * emission time to be associated with the request for tracking how far ahead 78 * of the GPU the submission is. 79 * 80 * When modifying this structure be very aware that we perform a lockless 81 * RCU lookup of it that may race against reallocation of the struct 82 * from the slab freelist. We intentionally do not zero the structure on 83 * allocation so that the lookup can use the dangling pointers (and is 84 * cogniscent that those pointers may be wrong). Instead, everything that 85 * needs to be initialised must be done so explicitly. 86 * 87 * The requests are reference counted. 88 */ 89 struct i915_request { 90 struct dma_fence fence; 91 spinlock_t lock; 92 93 /** On Which ring this request was generated */ 94 struct drm_i915_private *i915; 95 96 /** 97 * Context and ring buffer related to this request 98 * Contexts are refcounted, so when this request is associated with a 99 * context, we must increment the context's refcount, to guarantee that 100 * it persists while any request is linked to it. Requests themselves 101 * are also refcounted, so the request will only be freed when the last 102 * reference to it is dismissed, and the code in 103 * i915_request_free() will then decrement the refcount on the 104 * context. 105 */ 106 struct i915_gem_context *gem_context; 107 struct intel_engine_cs *engine; 108 struct intel_context *hw_context; 109 struct intel_ring *ring; 110 struct i915_timeline *timeline; 111 struct list_head signal_link; 112 113 /* 114 * The rcu epoch of when this request was allocated. Used to judiciously 115 * apply backpressure on future allocations to ensure that under 116 * mempressure there is sufficient RCU ticks for us to reclaim our 117 * RCU protected slabs. 118 */ 119 unsigned long rcustate; 120 121 /* 122 * Fences for the various phases in the request's lifetime. 123 * 124 * The submit fence is used to await upon all of the request's 125 * dependencies. When it is signaled, the request is ready to run. 126 * It is used by the driver to then queue the request for execution. 127 */ 128 struct i915_sw_fence submit; 129 wait_queue_entry_t submitq; 130 131 /* 132 * A list of everyone we wait upon, and everyone who waits upon us. 133 * Even though we will not be submitted to the hardware before the 134 * submit fence is signaled (it waits for all external events as well 135 * as our own requests), the scheduler still needs to know the 136 * dependency tree for the lifetime of the request (from execbuf 137 * to retirement), i.e. bidirectional dependency information for the 138 * request not tied to individual fences. 139 */ 140 struct i915_sched_node sched; 141 struct i915_dependency dep; 142 143 /* 144 * A convenience pointer to the current breadcrumb value stored in 145 * the HW status page (or our timeline's local equivalent). The full 146 * path would be rq->hw_context->ring->timeline->hwsp_seqno. 147 */ 148 const u32 *hwsp_seqno; 149 150 /** 151 * GEM sequence number associated with this request on the 152 * global execution timeline. It is zero when the request is not 153 * on the HW queue (i.e. not on the engine timeline list). 154 * Its value is guarded by the timeline spinlock. 155 */ 156 u32 global_seqno; 157 158 /** Position in the ring of the start of the request */ 159 u32 head; 160 161 /** Position in the ring of the start of the user packets */ 162 u32 infix; 163 164 /** 165 * Position in the ring of the start of the postfix. 166 * This is required to calculate the maximum available ring space 167 * without overwriting the postfix. 168 */ 169 u32 postfix; 170 171 /** Position in the ring of the end of the whole request */ 172 u32 tail; 173 174 /** Position in the ring of the end of any workarounds after the tail */ 175 u32 wa_tail; 176 177 /** Preallocate space in the ring for the emitting the request */ 178 u32 reserved_space; 179 180 /** Batch buffer related to this request if any (used for 181 * error state dump only). 182 */ 183 struct i915_vma *batch; 184 /** 185 * Additional buffers requested by userspace to be captured upon 186 * a GPU hang. The vma/obj on this list are protected by their 187 * active reference - all objects on this list must also be 188 * on the active_list (of their final request). 189 */ 190 struct i915_capture_list *capture_list; 191 struct list_head active_list; 192 193 /** Time at which this request was emitted, in jiffies. */ 194 unsigned long emitted_jiffies; 195 196 bool waitboost; 197 198 /** engine->request_list entry for this request */ 199 struct list_head link; 200 201 /** ring->request_list entry for this request */ 202 struct list_head ring_link; 203 204 struct drm_i915_file_private *file_priv; 205 /** file_priv list entry for this request */ 206 struct list_head client_link; 207 }; 208 209 #define I915_FENCE_GFP (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN) 210 211 extern const struct dma_fence_ops i915_fence_ops; 212 213 static inline bool dma_fence_is_i915(const struct dma_fence *fence) 214 { 215 return fence->ops == &i915_fence_ops; 216 } 217 218 struct i915_request * __must_check 219 i915_request_alloc(struct intel_engine_cs *engine, 220 struct i915_gem_context *ctx); 221 void i915_request_retire_upto(struct i915_request *rq); 222 223 static inline struct i915_request * 224 to_request(struct dma_fence *fence) 225 { 226 /* We assume that NULL fence/request are interoperable */ 227 BUILD_BUG_ON(offsetof(struct i915_request, fence) != 0); 228 GEM_BUG_ON(fence && !dma_fence_is_i915(fence)); 229 return container_of(fence, struct i915_request, fence); 230 } 231 232 static inline struct i915_request * 233 i915_request_get(struct i915_request *rq) 234 { 235 return to_request(dma_fence_get(&rq->fence)); 236 } 237 238 static inline struct i915_request * 239 i915_request_get_rcu(struct i915_request *rq) 240 { 241 return to_request(dma_fence_get_rcu(&rq->fence)); 242 } 243 244 static inline void 245 i915_request_put(struct i915_request *rq) 246 { 247 dma_fence_put(&rq->fence); 248 } 249 250 /** 251 * i915_request_global_seqno - report the current global seqno 252 * @request - the request 253 * 254 * A request is assigned a global seqno only when it is on the hardware 255 * execution queue. The global seqno can be used to maintain a list of 256 * requests on the same engine in retirement order, for example for 257 * constructing a priority queue for waiting. Prior to its execution, or 258 * if it is subsequently removed in the event of preemption, its global 259 * seqno is zero. As both insertion and removal from the execution queue 260 * may operate in IRQ context, it is not guarded by the usual struct_mutex 261 * BKL. Instead those relying on the global seqno must be prepared for its 262 * value to change between reads. Only when the request is complete can 263 * the global seqno be stable (due to the memory barriers on submitting 264 * the commands to the hardware to write the breadcrumb, if the HWS shows 265 * that it has passed the global seqno and the global seqno is unchanged 266 * after the read, it is indeed complete). 267 */ 268 static inline u32 269 i915_request_global_seqno(const struct i915_request *request) 270 { 271 return READ_ONCE(request->global_seqno); 272 } 273 274 int i915_request_await_object(struct i915_request *to, 275 struct drm_i915_gem_object *obj, 276 bool write); 277 int i915_request_await_dma_fence(struct i915_request *rq, 278 struct dma_fence *fence); 279 280 void i915_request_add(struct i915_request *rq); 281 282 void __i915_request_submit(struct i915_request *request); 283 void i915_request_submit(struct i915_request *request); 284 285 void i915_request_skip(struct i915_request *request, int error); 286 287 void __i915_request_unsubmit(struct i915_request *request); 288 void i915_request_unsubmit(struct i915_request *request); 289 290 /* Note: part of the intel_breadcrumbs family */ 291 bool i915_request_enable_breadcrumb(struct i915_request *request); 292 void i915_request_cancel_breadcrumb(struct i915_request *request); 293 294 long i915_request_wait(struct i915_request *rq, 295 unsigned int flags, 296 long timeout) 297 __attribute__((nonnull(1))); 298 #define I915_WAIT_INTERRUPTIBLE BIT(0) 299 #define I915_WAIT_LOCKED BIT(1) /* struct_mutex held, handle GPU reset */ 300 #define I915_WAIT_PRIORITY BIT(2) /* small priority bump for the request */ 301 #define I915_WAIT_ALL BIT(3) /* used by i915_gem_object_wait() */ 302 #define I915_WAIT_FOR_IDLE_BOOST BIT(4) 303 304 static inline bool i915_request_signaled(const struct i915_request *rq) 305 { 306 /* The request may live longer than its HWSP, so check flags first! */ 307 return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags); 308 } 309 310 static inline bool i915_request_is_active(const struct i915_request *rq) 311 { 312 return test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags); 313 } 314 315 /** 316 * Returns true if seq1 is later than seq2. 317 */ 318 static inline bool i915_seqno_passed(u32 seq1, u32 seq2) 319 { 320 return (s32)(seq1 - seq2) >= 0; 321 } 322 323 static inline u32 __hwsp_seqno(const struct i915_request *rq) 324 { 325 return READ_ONCE(*rq->hwsp_seqno); 326 } 327 328 /** 329 * hwsp_seqno - the current breadcrumb value in the HW status page 330 * @rq: the request, to chase the relevant HW status page 331 * 332 * The emphasis in naming here is that hwsp_seqno() is not a property of the 333 * request, but an indication of the current HW state (associated with this 334 * request). Its value will change as the GPU executes more requests. 335 * 336 * Returns the current breadcrumb value in the associated HW status page (or 337 * the local timeline's equivalent) for this request. The request itself 338 * has the associated breadcrumb value of rq->fence.seqno, when the HW 339 * status page has that breadcrumb or later, this request is complete. 340 */ 341 static inline u32 hwsp_seqno(const struct i915_request *rq) 342 { 343 u32 seqno; 344 345 rcu_read_lock(); /* the HWSP may be freed at runtime */ 346 seqno = __hwsp_seqno(rq); 347 rcu_read_unlock(); 348 349 return seqno; 350 } 351 352 static inline bool __i915_request_has_started(const struct i915_request *rq) 353 { 354 return i915_seqno_passed(hwsp_seqno(rq), rq->fence.seqno - 1); 355 } 356 357 /** 358 * i915_request_started - check if the request has begun being executed 359 * @rq: the request 360 * 361 * Returns true if the request has been submitted to hardware, and the hardware 362 * has advanced passed the end of the previous request and so should be either 363 * currently processing the request (though it may be preempted and so 364 * not necessarily the next request to complete) or have completed the request. 365 */ 366 static inline bool i915_request_started(const struct i915_request *rq) 367 { 368 if (i915_request_signaled(rq)) 369 return true; 370 371 /* Remember: started but may have since been preempted! */ 372 return __i915_request_has_started(rq); 373 } 374 375 /** 376 * i915_request_is_running - check if the request may actually be executing 377 * @rq: the request 378 * 379 * Returns true if the request is currently submitted to hardware, has passed 380 * its start point (i.e. the context is setup and not busywaiting). Note that 381 * it may no longer be running by the time the function returns! 382 */ 383 static inline bool i915_request_is_running(const struct i915_request *rq) 384 { 385 if (!i915_request_is_active(rq)) 386 return false; 387 388 return __i915_request_has_started(rq); 389 } 390 391 static inline bool i915_request_completed(const struct i915_request *rq) 392 { 393 if (i915_request_signaled(rq)) 394 return true; 395 396 return i915_seqno_passed(hwsp_seqno(rq), rq->fence.seqno); 397 } 398 399 static inline void i915_request_mark_complete(struct i915_request *rq) 400 { 401 rq->hwsp_seqno = (u32 *)&rq->fence.seqno; /* decouple from HWSP */ 402 } 403 404 void i915_retire_requests(struct drm_i915_private *i915); 405 406 #endif /* I915_REQUEST_H */ 407