1 /* 2 * Copyright © 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/kthread.h> 26 #include <trace/events/dma_fence.h> 27 #include <uapi/linux/sched/types.h> 28 29 #include "i915_drv.h" 30 #include "i915_trace.h" 31 32 static void irq_enable(struct intel_engine_cs *engine) 33 { 34 if (!engine->irq_enable) 35 return; 36 37 /* Caller disables interrupts */ 38 spin_lock(&engine->gt->irq_lock); 39 engine->irq_enable(engine); 40 spin_unlock(&engine->gt->irq_lock); 41 } 42 43 static void irq_disable(struct intel_engine_cs *engine) 44 { 45 if (!engine->irq_disable) 46 return; 47 48 /* Caller disables interrupts */ 49 spin_lock(&engine->gt->irq_lock); 50 engine->irq_disable(engine); 51 spin_unlock(&engine->gt->irq_lock); 52 } 53 54 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) 55 { 56 lockdep_assert_held(&b->irq_lock); 57 58 GEM_BUG_ON(!b->irq_enabled); 59 if (!--b->irq_enabled) 60 irq_disable(container_of(b, 61 struct intel_engine_cs, 62 breadcrumbs)); 63 64 b->irq_armed = false; 65 } 66 67 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine) 68 { 69 struct intel_breadcrumbs *b = &engine->breadcrumbs; 70 unsigned long flags; 71 72 if (!b->irq_armed) 73 return; 74 75 spin_lock_irqsave(&b->irq_lock, flags); 76 if (b->irq_armed) 77 __intel_breadcrumbs_disarm_irq(b); 78 spin_unlock_irqrestore(&b->irq_lock, flags); 79 } 80 81 static inline bool __request_completed(const struct i915_request *rq) 82 { 83 return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno); 84 } 85 86 __maybe_unused static bool 87 check_signal_order(struct intel_context *ce, struct i915_request *rq) 88 { 89 if (!list_is_last(&rq->signal_link, &ce->signals) && 90 i915_seqno_passed(rq->fence.seqno, 91 list_next_entry(rq, signal_link)->fence.seqno)) 92 return false; 93 94 if (!list_is_first(&rq->signal_link, &ce->signals) && 95 i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno, 96 rq->fence.seqno)) 97 return false; 98 99 return true; 100 } 101 102 static bool 103 __dma_fence_signal(struct dma_fence *fence) 104 { 105 return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags); 106 } 107 108 static void 109 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp) 110 { 111 fence->timestamp = timestamp; 112 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); 113 trace_dma_fence_signaled(fence); 114 } 115 116 static void 117 __dma_fence_signal__notify(struct dma_fence *fence, 118 const struct list_head *list) 119 { 120 struct dma_fence_cb *cur, *tmp; 121 122 lockdep_assert_held(fence->lock); 123 lockdep_assert_irqs_disabled(); 124 125 list_for_each_entry_safe(cur, tmp, list, node) { 126 INIT_LIST_HEAD(&cur->node); 127 cur->func(fence, cur); 128 } 129 } 130 131 void intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine) 132 { 133 struct intel_breadcrumbs *b = &engine->breadcrumbs; 134 const ktime_t timestamp = ktime_get(); 135 struct intel_context *ce, *cn; 136 struct list_head *pos, *next; 137 LIST_HEAD(signal); 138 139 spin_lock(&b->irq_lock); 140 141 if (b->irq_armed && list_empty(&b->signalers)) 142 __intel_breadcrumbs_disarm_irq(b); 143 144 list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) { 145 GEM_BUG_ON(list_empty(&ce->signals)); 146 147 list_for_each_safe(pos, next, &ce->signals) { 148 struct i915_request *rq = 149 list_entry(pos, typeof(*rq), signal_link); 150 151 GEM_BUG_ON(!check_signal_order(ce, rq)); 152 153 if (!__request_completed(rq)) 154 break; 155 156 GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, 157 &rq->fence.flags)); 158 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 159 160 if (!__dma_fence_signal(&rq->fence)) 161 continue; 162 163 /* 164 * Queue for execution after dropping the signaling 165 * spinlock as the callback chain may end up adding 166 * more signalers to the same context or engine. 167 */ 168 i915_request_get(rq); 169 list_add_tail(&rq->signal_link, &signal); 170 } 171 172 /* 173 * We process the list deletion in bulk, only using a list_add 174 * (not list_move) above but keeping the status of 175 * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit. 176 */ 177 if (!list_is_first(pos, &ce->signals)) { 178 /* Advance the list to the first incomplete request */ 179 __list_del_many(&ce->signals, pos); 180 if (&ce->signals == pos) /* now empty */ 181 list_del_init(&ce->signal_link); 182 } 183 } 184 185 spin_unlock(&b->irq_lock); 186 187 list_for_each_safe(pos, next, &signal) { 188 struct i915_request *rq = 189 list_entry(pos, typeof(*rq), signal_link); 190 struct list_head cb_list; 191 192 spin_lock(&rq->lock); 193 list_replace(&rq->fence.cb_list, &cb_list); 194 __dma_fence_signal__timestamp(&rq->fence, timestamp); 195 __dma_fence_signal__notify(&rq->fence, &cb_list); 196 spin_unlock(&rq->lock); 197 198 i915_request_put(rq); 199 } 200 } 201 202 void intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine) 203 { 204 local_irq_disable(); 205 intel_engine_breadcrumbs_irq(engine); 206 local_irq_enable(); 207 } 208 209 static void signal_irq_work(struct irq_work *work) 210 { 211 struct intel_engine_cs *engine = 212 container_of(work, typeof(*engine), breadcrumbs.irq_work); 213 214 intel_engine_breadcrumbs_irq(engine); 215 } 216 217 static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) 218 { 219 struct intel_engine_cs *engine = 220 container_of(b, struct intel_engine_cs, breadcrumbs); 221 222 lockdep_assert_held(&b->irq_lock); 223 if (b->irq_armed) 224 return; 225 226 /* 227 * The breadcrumb irq will be disarmed on the interrupt after the 228 * waiters are signaled. This gives us a single interrupt window in 229 * which we can add a new waiter and avoid the cost of re-enabling 230 * the irq. 231 */ 232 b->irq_armed = true; 233 234 /* 235 * Since we are waiting on a request, the GPU should be busy 236 * and should have its own rpm reference. This is tracked 237 * by i915->gt.awake, we can forgo holding our own wakref 238 * for the interrupt as before i915->gt.awake is released (when 239 * the driver is idle) we disarm the breadcrumbs. 240 */ 241 242 if (!b->irq_enabled++) 243 irq_enable(engine); 244 } 245 246 void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) 247 { 248 struct intel_breadcrumbs *b = &engine->breadcrumbs; 249 250 spin_lock_init(&b->irq_lock); 251 INIT_LIST_HEAD(&b->signalers); 252 253 init_irq_work(&b->irq_work, signal_irq_work); 254 } 255 256 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) 257 { 258 struct intel_breadcrumbs *b = &engine->breadcrumbs; 259 unsigned long flags; 260 261 spin_lock_irqsave(&b->irq_lock, flags); 262 263 if (b->irq_enabled) 264 irq_enable(engine); 265 else 266 irq_disable(engine); 267 268 spin_unlock_irqrestore(&b->irq_lock, flags); 269 } 270 271 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) 272 { 273 } 274 275 bool i915_request_enable_breadcrumb(struct i915_request *rq) 276 { 277 lockdep_assert_held(&rq->lock); 278 lockdep_assert_irqs_disabled(); 279 280 if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) { 281 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; 282 struct intel_context *ce = rq->hw_context; 283 struct list_head *pos; 284 285 spin_lock(&b->irq_lock); 286 GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)); 287 288 __intel_breadcrumbs_arm_irq(b); 289 290 /* 291 * We keep the seqno in retirement order, so we can break 292 * inside intel_engine_breadcrumbs_irq as soon as we've passed 293 * the last completed request (or seen a request that hasn't 294 * event started). We could iterate the timeline->requests list, 295 * but keeping a separate signalers_list has the advantage of 296 * hopefully being much smaller than the full list and so 297 * provides faster iteration and detection when there are no 298 * more interrupts required for this context. 299 * 300 * We typically expect to add new signalers in order, so we 301 * start looking for our insertion point from the tail of 302 * the list. 303 */ 304 list_for_each_prev(pos, &ce->signals) { 305 struct i915_request *it = 306 list_entry(pos, typeof(*it), signal_link); 307 308 if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno)) 309 break; 310 } 311 list_add(&rq->signal_link, pos); 312 if (pos == &ce->signals) /* catch transitions from empty list */ 313 list_move_tail(&ce->signal_link, &b->signalers); 314 GEM_BUG_ON(!check_signal_order(ce, rq)); 315 316 set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 317 spin_unlock(&b->irq_lock); 318 } 319 320 return !__request_completed(rq); 321 } 322 323 void i915_request_cancel_breadcrumb(struct i915_request *rq) 324 { 325 struct intel_breadcrumbs *b = &rq->engine->breadcrumbs; 326 327 lockdep_assert_held(&rq->lock); 328 lockdep_assert_irqs_disabled(); 329 330 /* 331 * We must wait for b->irq_lock so that we know the interrupt handler 332 * has released its reference to the intel_context and has completed 333 * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if 334 * required). 335 */ 336 spin_lock(&b->irq_lock); 337 if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) { 338 struct intel_context *ce = rq->hw_context; 339 340 list_del(&rq->signal_link); 341 if (list_empty(&ce->signals)) 342 list_del_init(&ce->signal_link); 343 344 clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); 345 } 346 spin_unlock(&b->irq_lock); 347 } 348 349 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, 350 struct drm_printer *p) 351 { 352 struct intel_breadcrumbs *b = &engine->breadcrumbs; 353 struct intel_context *ce; 354 struct i915_request *rq; 355 356 if (list_empty(&b->signalers)) 357 return; 358 359 drm_printf(p, "Signals:\n"); 360 361 spin_lock_irq(&b->irq_lock); 362 list_for_each_entry(ce, &b->signalers, signal_link) { 363 list_for_each_entry(rq, &ce->signals, signal_link) { 364 drm_printf(p, "\t[%llx:%llx%s] @ %dms\n", 365 rq->fence.context, rq->fence.seqno, 366 i915_request_completed(rq) ? "!" : 367 i915_request_started(rq) ? "*" : 368 "", 369 jiffies_to_msecs(jiffies - rq->emitted_jiffies)); 370 } 371 } 372 spin_unlock_irq(&b->irq_lock); 373 } 374