1 /* 2 * Copyright © 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/kthread.h> 26 #include <trace/events/dma_fence.h> 27 #include <uapi/linux/sched/types.h> 28 29 #include "i915_drv.h" 30 31 static void irq_enable(struct intel_engine_cs *engine) 32 { 33 if (!engine->irq_enable) 34 return; 35 36 /* Caller disables interrupts */ 37 spin_lock(&engine->i915->irq_lock); 38 engine->irq_enable(engine); 39 spin_unlock(&engine->i915->irq_lock); 40 } 41 42 static void irq_disable(struct intel_engine_cs *engine) 43 { 44 if (!engine->irq_disable) 45 return; 46 47 /* Caller disables interrupts */ 48 spin_lock(&engine->i915->irq_lock); 49 engine->irq_disable(engine); 50 spin_unlock(&engine->i915->irq_lock); 51 } 52 53 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) 54 { 55 lockdep_assert_held(&b->irq_lock); 56 57 GEM_BUG_ON(!b->irq_enabled); 58 if (!--b->irq_enabled) 59 irq_disable(container_of(b, 60 struct intel_engine_cs, 61 breadcrumbs)); 62 63 b->irq_armed = false; 64 } 65 66 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine) 67 { 68 struct intel_breadcrumbs *b = &engine->breadcrumbs; 69 70 if (!b->irq_armed) 71 return; 72 73 spin_lock_irq(&b->irq_lock); 74 if (b->irq_armed) 75 __intel_breadcrumbs_disarm_irq(b); 76 spin_unlock_irq(&b->irq_lock); 77 } 78 79 static inline bool __request_completed(const struct i915_request *rq) 80 { 81 return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno); 82 } 83 84 __maybe_unused static bool 85 check_signal_order(struct intel_context *ce, struct i915_request *rq) 86 { 87 if (!list_is_last(&rq->signal_link, &ce->signals) && 88 i915_seqno_passed(rq->fence.seqno, 89 list_next_entry(rq, signal_link)->fence.seqno)) 90 return false; 91 92 if (!list_is_first(&rq->signal_link, &ce->signals) && 93 i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno, 94 rq->fence.seqno)) 95 return false; 96 97 return true; 98 } 99 100 static bool 101 __dma_fence_signal(struct dma_fence *fence) 102 { 103 return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags); 104 } 105 106 static void 107 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp) 108 { 109 fence->timestamp = timestamp; 110 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); 111 trace_dma_fence_signaled(fence); 112 } 113 114 static void 115 __dma_fence_signal__notify(struct dma_fence *fence, 116 const struct list_head *list) 117 { 118 struct dma_fence_cb *cur, *tmp; 119 120 lockdep_assert_held(fence->lock); 121 lockdep_assert_irqs_disabled(); 122 123 list_for_each_entry_safe(cur, tmp, list, node) { 124 INIT_LIST_HEAD(&cur->node); 125 cur->func(fence, cur); 126 } 127 } 128 129 void intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine) 130 { 131 struct intel_breadcrumbs *b = &engine->breadcrumbs; 132 const ktime_t timestamp = ktime_get(); 133 struct intel_context *ce, *cn; 134 struct list_head *pos, *next; 135 LIST_HEAD(signal); 136 137 spin_lock(&b->irq_lock); 138 139 if (b->irq_armed && list_empty(&b->signalers)) 140 __intel_breadcrumbs_disarm_irq(b); 141 142 list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) { 143 GEM_BUG_ON(list_empty(&ce->signals)); 144 145 list_for_each_safe(pos, next, &ce->signals) { 146 struct i915_request *rq = 147 list_entry(pos, typeof(*rq), signal_link); 148 149 GEM_BUG_ON(!check_signal_order(ce, rq)); 150 151 if (!__request_completed(rq)) 152 break; 153 154 GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, 155 &rq->fence.flags)); 156 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 157 158 if (!__dma_fence_signal(&rq->fence)) 159 continue; 160 161 /* 162 * Queue for execution after dropping the signaling 163 * spinlock as the callback chain may end up adding 164 * more signalers to the same context or engine. 165 */ 166 i915_request_get(rq); 167 list_add_tail(&rq->signal_link, &signal); 168 } 169 170 /* 171 * We process the list deletion in bulk, only using a list_add 172 * (not list_move) above but keeping the status of 173 * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit. 174 */ 175 if (!list_is_first(pos, &ce->signals)) { 176 /* Advance the list to the first incomplete request */ 177 __list_del_many(&ce->signals, pos); 178 if (&ce->signals == pos) /* now empty */ 179 list_del_init(&ce->signal_link); 180 } 181 } 182 183 spin_unlock(&b->irq_lock); 184 185 list_for_each_safe(pos, next, &signal) { 186 struct i915_request *rq = 187 list_entry(pos, typeof(*rq), signal_link); 188 struct list_head cb_list; 189 190 spin_lock(&rq->lock); 191 list_replace(&rq->fence.cb_list, &cb_list); 192 __dma_fence_signal__timestamp(&rq->fence, timestamp); 193 __dma_fence_signal__notify(&rq->fence, &cb_list); 194 spin_unlock(&rq->lock); 195 196 i915_request_put(rq); 197 } 198 } 199 200 void intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine) 201 { 202 local_irq_disable(); 203 intel_engine_breadcrumbs_irq(engine); 204 local_irq_enable(); 205 } 206 207 static void signal_irq_work(struct irq_work *work) 208 { 209 struct intel_engine_cs *engine = 210 container_of(work, typeof(*engine), breadcrumbs.irq_work); 211 212 intel_engine_breadcrumbs_irq(engine); 213 } 214 215 void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine) 216 { 217 struct intel_breadcrumbs *b = &engine->breadcrumbs; 218 219 spin_lock_irq(&b->irq_lock); 220 if (!b->irq_enabled++) 221 irq_enable(engine); 222 GEM_BUG_ON(!b->irq_enabled); /* no overflow! */ 223 spin_unlock_irq(&b->irq_lock); 224 } 225 226 void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine) 227 { 228 struct intel_breadcrumbs *b = &engine->breadcrumbs; 229 230 spin_lock_irq(&b->irq_lock); 231 GEM_BUG_ON(!b->irq_enabled); /* no underflow! */ 232 if (!--b->irq_enabled) 233 irq_disable(engine); 234 spin_unlock_irq(&b->irq_lock); 235 } 236 237 static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) 238 { 239 struct intel_engine_cs *engine = 240 container_of(b, struct intel_engine_cs, breadcrumbs); 241 242 lockdep_assert_held(&b->irq_lock); 243 if (b->irq_armed) 244 return; 245 246 /* 247 * The breadcrumb irq will be disarmed on the interrupt after the 248 * waiters are signaled. This gives us a single interrupt window in 249 * which we can add a new waiter and avoid the cost of re-enabling 250 * the irq. 251 */ 252 b->irq_armed = true; 253 254 /* 255 * Since we are waiting on a request, the GPU should be busy 256 * and should have its own rpm reference. This is tracked 257 * by i915->gt.awake, we can forgo holding our own wakref 258 * for the interrupt as before i915->gt.awake is released (when 259 * the driver is idle) we disarm the breadcrumbs. 260 */ 261 262 if (!b->irq_enabled++) 263 irq_enable(engine); 264 } 265 266 void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) 267 { 268 struct intel_breadcrumbs *b = &engine->breadcrumbs; 269 270 spin_lock_init(&b->irq_lock); 271 INIT_LIST_HEAD(&b->signalers); 272 273 init_irq_work(&b->irq_work, signal_irq_work); 274 } 275 276 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) 277 { 278 struct intel_breadcrumbs *b = &engine->breadcrumbs; 279 unsigned long flags; 280 281 spin_lock_irqsave(&b->irq_lock, flags); 282 283 if (b->irq_enabled) 284 irq_enable(engine); 285 else 286 irq_disable(engine); 287 288 spin_unlock_irqrestore(&b->irq_lock, flags); 289 } 290 291 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) 292 { 293 } 294 295 bool i915_request_enable_breadcrumb(struct i915_request *rq) 296 { 297 lockdep_assert_held(&rq->lock); 298 lockdep_assert_irqs_disabled(); 299 300 if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) { 301 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; 302 struct intel_context *ce = rq->hw_context; 303 struct list_head *pos; 304 305 spin_lock(&b->irq_lock); 306 GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)); 307 308 __intel_breadcrumbs_arm_irq(b); 309 310 /* 311 * We keep the seqno in retirement order, so we can break 312 * inside intel_engine_breadcrumbs_irq as soon as we've passed 313 * the last completed request (or seen a request that hasn't 314 * event started). We could iterate the timeline->requests list, 315 * but keeping a separate signalers_list has the advantage of 316 * hopefully being much smaller than the full list and so 317 * provides faster iteration and detection when there are no 318 * more interrupts required for this context. 319 * 320 * We typically expect to add new signalers in order, so we 321 * start looking for our insertion point from the tail of 322 * the list. 323 */ 324 list_for_each_prev(pos, &ce->signals) { 325 struct i915_request *it = 326 list_entry(pos, typeof(*it), signal_link); 327 328 if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno)) 329 break; 330 } 331 list_add(&rq->signal_link, pos); 332 if (pos == &ce->signals) /* catch transitions from empty list */ 333 list_move_tail(&ce->signal_link, &b->signalers); 334 GEM_BUG_ON(!check_signal_order(ce, rq)); 335 336 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 337 spin_unlock(&b->irq_lock); 338 } 339 340 return !__request_completed(rq); 341 } 342 343 void i915_request_cancel_breadcrumb(struct i915_request *rq) 344 { 345 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; 346 347 lockdep_assert_held(&rq->lock); 348 lockdep_assert_irqs_disabled(); 349 350 /* 351 * We must wait for b->irq_lock so that we know the interrupt handler 352 * has released its reference to the intel_context and has completed 353 * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if 354 * required). 355 */ 356 spin_lock(&b->irq_lock); 357 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) { 358 struct intel_context *ce = rq->hw_context; 359 360 list_del(&rq->signal_link); 361 if (list_empty(&ce->signals)) 362 list_del_init(&ce->signal_link); 363 364 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 365 } 366 spin_unlock(&b->irq_lock); 367 } 368 369 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, 370 struct drm_printer *p) 371 { 372 struct intel_breadcrumbs *b = &engine->breadcrumbs; 373 struct intel_context *ce; 374 struct i915_request *rq; 375 376 if (list_empty(&b->signalers)) 377 return; 378 379 drm_printf(p, "Signals:\n"); 380 381 spin_lock_irq(&b->irq_lock); 382 list_for_each_entry(ce, &b->signalers, signal_link) { 383 list_for_each_entry(rq, &ce->signals, signal_link) { 384 drm_printf(p, "\t[%llx:%llx%s] @ %dms\n", 385 rq->fence.context, rq->fence.seqno, 386 i915_request_completed(rq) ? "!" : 387 i915_request_started(rq) ? "*" : 388 "", 389 jiffies_to_msecs(jiffies - rq->emitted_jiffies)); 390 } 391 } 392 spin_unlock_irq(&b->irq_lock); 393 } 394