1 /* 2 * Copyright © 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/kthread.h> 26 #include <trace/events/dma_fence.h> 27 #include <uapi/linux/sched/types.h> 28 29 #include "i915_drv.h" 30 #include "i915_trace.h" 31 #include "intel_gt_pm.h" 32 #include "intel_gt_requests.h" 33 34 static void irq_enable(struct intel_engine_cs *engine) 35 { 36 if (!engine->irq_enable) 37 return; 38 39 /* Caller disables interrupts */ 40 spin_lock(&engine->gt->irq_lock); 41 engine->irq_enable(engine); 42 spin_unlock(&engine->gt->irq_lock); 43 } 44 45 static void irq_disable(struct intel_engine_cs *engine) 46 { 47 if (!engine->irq_disable) 48 return; 49 50 /* Caller disables interrupts */ 51 spin_lock(&engine->gt->irq_lock); 52 engine->irq_disable(engine); 53 spin_unlock(&engine->gt->irq_lock); 54 } 55 56 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) 57 { 58 struct intel_engine_cs *engine = 59 container_of(b, struct intel_engine_cs, breadcrumbs); 60 61 lockdep_assert_held(&b->irq_lock); 62 63 GEM_BUG_ON(!b->irq_enabled); 64 if (!--b->irq_enabled) 65 irq_disable(engine); 66 67 WRITE_ONCE(b->irq_armed, false); 68 intel_gt_pm_put_async(engine->gt); 69 } 70 71 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine) 72 { 73 struct intel_breadcrumbs *b = &engine->breadcrumbs; 74 unsigned long flags; 75 76 if (!READ_ONCE(b->irq_armed)) 77 return; 78 79 spin_lock_irqsave(&b->irq_lock, flags); 80 if (b->irq_armed) 81 __intel_breadcrumbs_disarm_irq(b); 82 spin_unlock_irqrestore(&b->irq_lock, flags); 83 } 84 85 static inline bool __request_completed(const struct i915_request *rq) 86 { 87 return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno); 88 } 89 90 __maybe_unused static bool 91 check_signal_order(struct intel_context *ce, struct i915_request *rq) 92 { 93 if (!list_is_last(&rq->signal_link, &ce->signals) && 94 i915_seqno_passed(rq->fence.seqno, 95 list_next_entry(rq, signal_link)->fence.seqno)) 96 return false; 97 98 if (!list_is_first(&rq->signal_link, &ce->signals) && 99 i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno, 100 rq->fence.seqno)) 101 return false; 102 103 return true; 104 } 105 106 static bool 107 __dma_fence_signal(struct dma_fence *fence) 108 { 109 return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags); 110 } 111 112 static void 113 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp) 114 { 115 fence->timestamp = timestamp; 116 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); 117 trace_dma_fence_signaled(fence); 118 } 119 120 static void 121 __dma_fence_signal__notify(struct dma_fence *fence, 122 const struct list_head *list) 123 { 124 struct dma_fence_cb *cur, *tmp; 125 126 lockdep_assert_held(fence->lock); 127 128 list_for_each_entry_safe(cur, tmp, list, node) { 129 INIT_LIST_HEAD(&cur->node); 130 cur->func(fence, cur); 131 } 132 } 133 134 static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl) 135 { 136 struct intel_engine_cs *engine = 137 container_of(b, struct intel_engine_cs, breadcrumbs); 138 139 if (unlikely(intel_engine_is_virtual(engine))) 140 engine = intel_virtual_engine_get_sibling(engine, 0); 141 142 intel_engine_add_retire(engine, tl); 143 } 144 145 static void __signal_request(struct i915_request *rq, struct list_head *signals) 146 { 147 GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)); 148 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 149 150 if (!__dma_fence_signal(&rq->fence)) 151 return; 152 153 i915_request_get(rq); 154 list_add_tail(&rq->signal_link, signals); 155 } 156 157 static void signal_irq_work(struct irq_work *work) 158 { 159 struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work); 160 const ktime_t timestamp = ktime_get(); 161 struct intel_context *ce, *cn; 162 struct list_head *pos, *next; 163 LIST_HEAD(signal); 164 165 spin_lock(&b->irq_lock); 166 167 if (b->irq_armed && list_empty(&b->signalers)) 168 __intel_breadcrumbs_disarm_irq(b); 169 170 list_splice_init(&b->signaled_requests, &signal); 171 172 list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) { 173 GEM_BUG_ON(list_empty(&ce->signals)); 174 175 list_for_each_safe(pos, next, &ce->signals) { 176 struct i915_request *rq = 177 list_entry(pos, typeof(*rq), signal_link); 178 179 GEM_BUG_ON(!check_signal_order(ce, rq)); 180 if (!__request_completed(rq)) 181 break; 182 183 /* 184 * Queue for execution after dropping the signaling 185 * spinlock as the callback chain may end up adding 186 * more signalers to the same context or engine. 187 */ 188 __signal_request(rq, &signal); 189 } 190 191 /* 192 * We process the list deletion in bulk, only using a list_add 193 * (not list_move) above but keeping the status of 194 * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit. 195 */ 196 if (!list_is_first(pos, &ce->signals)) { 197 /* Advance the list to the first incomplete request */ 198 __list_del_many(&ce->signals, pos); 199 if (&ce->signals == pos) { /* now empty */ 200 list_del_init(&ce->signal_link); 201 add_retire(b, ce->timeline); 202 } 203 } 204 } 205 206 spin_unlock(&b->irq_lock); 207 208 list_for_each_safe(pos, next, &signal) { 209 struct i915_request *rq = 210 list_entry(pos, typeof(*rq), signal_link); 211 struct list_head cb_list; 212 213 spin_lock(&rq->lock); 214 list_replace(&rq->fence.cb_list, &cb_list); 215 __dma_fence_signal__timestamp(&rq->fence, timestamp); 216 __dma_fence_signal__notify(&rq->fence, &cb_list); 217 spin_unlock(&rq->lock); 218 219 i915_request_put(rq); 220 } 221 } 222 223 static bool __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) 224 { 225 struct intel_engine_cs *engine = 226 container_of(b, struct intel_engine_cs, breadcrumbs); 227 228 lockdep_assert_held(&b->irq_lock); 229 if (b->irq_armed) 230 return true; 231 232 if (!intel_gt_pm_get_if_awake(engine->gt)) 233 return false; 234 235 /* 236 * The breadcrumb irq will be disarmed on the interrupt after the 237 * waiters are signaled. This gives us a single interrupt window in 238 * which we can add a new waiter and avoid the cost of re-enabling 239 * the irq. 240 */ 241 WRITE_ONCE(b->irq_armed, true); 242 243 /* 244 * Since we are waiting on a request, the GPU should be busy 245 * and should have its own rpm reference. This is tracked 246 * by i915->gt.awake, we can forgo holding our own wakref 247 * for the interrupt as before i915->gt.awake is released (when 248 * the driver is idle) we disarm the breadcrumbs. 249 */ 250 251 if (!b->irq_enabled++) 252 irq_enable(engine); 253 254 return true; 255 } 256 257 void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) 258 { 259 struct intel_breadcrumbs *b = &engine->breadcrumbs; 260 261 spin_lock_init(&b->irq_lock); 262 INIT_LIST_HEAD(&b->signalers); 263 INIT_LIST_HEAD(&b->signaled_requests); 264 265 init_irq_work(&b->irq_work, signal_irq_work); 266 } 267 268 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) 269 { 270 struct intel_breadcrumbs *b = &engine->breadcrumbs; 271 unsigned long flags; 272 273 spin_lock_irqsave(&b->irq_lock, flags); 274 275 if (b->irq_enabled) 276 irq_enable(engine); 277 else 278 irq_disable(engine); 279 280 spin_unlock_irqrestore(&b->irq_lock, flags); 281 } 282 283 void intel_engine_transfer_stale_breadcrumbs(struct intel_engine_cs *engine, 284 struct intel_context *ce) 285 { 286 struct intel_breadcrumbs *b = &engine->breadcrumbs; 287 unsigned long flags; 288 289 spin_lock_irqsave(&b->irq_lock, flags); 290 if (!list_empty(&ce->signals)) { 291 struct i915_request *rq, *next; 292 293 /* Queue for executing the signal callbacks in the irq_work */ 294 list_for_each_entry_safe(rq, next, &ce->signals, signal_link) { 295 GEM_BUG_ON(rq->engine != engine); 296 GEM_BUG_ON(!__request_completed(rq)); 297 298 __signal_request(rq, &b->signaled_requests); 299 } 300 301 INIT_LIST_HEAD(&ce->signals); 302 list_del_init(&ce->signal_link); 303 304 irq_work_queue(&b->irq_work); 305 } 306 spin_unlock_irqrestore(&b->irq_lock, flags); 307 } 308 309 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) 310 { 311 } 312 313 bool i915_request_enable_breadcrumb(struct i915_request *rq) 314 { 315 lockdep_assert_held(&rq->lock); 316 317 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags)) 318 return true; 319 320 if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) { 321 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; 322 struct intel_context *ce = rq->context; 323 struct list_head *pos; 324 325 spin_lock(&b->irq_lock); 326 327 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) 328 goto unlock; 329 330 if (!__intel_breadcrumbs_arm_irq(b)) 331 goto unlock; 332 333 /* 334 * We keep the seqno in retirement order, so we can break 335 * inside intel_engine_signal_breadcrumbs as soon as we've 336 * passed the last completed request (or seen a request that 337 * hasn't event started). We could walk the timeline->requests, 338 * but keeping a separate signalers_list has the advantage of 339 * hopefully being much smaller than the full list and so 340 * provides faster iteration and detection when there are no 341 * more interrupts required for this context. 342 * 343 * We typically expect to add new signalers in order, so we 344 * start looking for our insertion point from the tail of 345 * the list. 346 */ 347 list_for_each_prev(pos, &ce->signals) { 348 struct i915_request *it = 349 list_entry(pos, typeof(*it), signal_link); 350 351 if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno)) 352 break; 353 } 354 list_add(&rq->signal_link, pos); 355 if (pos == &ce->signals) /* catch transitions from empty list */ 356 list_move_tail(&ce->signal_link, &b->signalers); 357 GEM_BUG_ON(!check_signal_order(ce, rq)); 358 359 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 360 unlock: 361 spin_unlock(&b->irq_lock); 362 } 363 364 return !__request_completed(rq); 365 } 366 367 void i915_request_cancel_breadcrumb(struct i915_request *rq) 368 { 369 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; 370 371 lockdep_assert_held(&rq->lock); 372 373 /* 374 * We must wait for b->irq_lock so that we know the interrupt handler 375 * has released its reference to the intel_context and has completed 376 * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if 377 * required). 378 */ 379 spin_lock(&b->irq_lock); 380 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) { 381 struct intel_context *ce = rq->context; 382 383 list_del(&rq->signal_link); 384 if (list_empty(&ce->signals)) 385 list_del_init(&ce->signal_link); 386 387 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 388 } 389 spin_unlock(&b->irq_lock); 390 } 391 392 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, 393 struct drm_printer *p) 394 { 395 struct intel_breadcrumbs *b = &engine->breadcrumbs; 396 struct intel_context *ce; 397 struct i915_request *rq; 398 399 if (list_empty(&b->signalers)) 400 return; 401 402 drm_printf(p, "Signals:\n"); 403 404 spin_lock_irq(&b->irq_lock); 405 list_for_each_entry(ce, &b->signalers, signal_link) { 406 list_for_each_entry(rq, &ce->signals, signal_link) { 407 drm_printf(p, "\t[%llx:%llx%s] @ %dms\n", 408 rq->fence.context, rq->fence.seqno, 409 i915_request_completed(rq) ? "!" : 410 i915_request_started(rq) ? "*" : 411 "", 412 jiffies_to_msecs(jiffies - rq->emitted_jiffies)); 413 } 414 } 415 spin_unlock_irq(&b->irq_lock); 416 } 417