1 /*
2  * Copyright © 2015 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/kthread.h>
26 #include <trace/events/dma_fence.h>
27 #include <uapi/linux/sched/types.h>
28 
29 #include "i915_drv.h"
30 
31 static void irq_enable(struct intel_engine_cs *engine)
32 {
33 	if (!engine->irq_enable)
34 		return;
35 
36 	/* Caller disables interrupts */
37 	spin_lock(&engine->i915->irq_lock);
38 	engine->irq_enable(engine);
39 	spin_unlock(&engine->i915->irq_lock);
40 }
41 
42 static void irq_disable(struct intel_engine_cs *engine)
43 {
44 	if (!engine->irq_disable)
45 		return;
46 
47 	/* Caller disables interrupts */
48 	spin_lock(&engine->i915->irq_lock);
49 	engine->irq_disable(engine);
50 	spin_unlock(&engine->i915->irq_lock);
51 }
52 
53 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
54 {
55 	lockdep_assert_held(&b->irq_lock);
56 
57 	GEM_BUG_ON(!b->irq_enabled);
58 	if (!--b->irq_enabled)
59 		irq_disable(container_of(b,
60 					 struct intel_engine_cs,
61 					 breadcrumbs));
62 
63 	b->irq_armed = false;
64 }
65 
66 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
67 {
68 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
69 
70 	if (!b->irq_armed)
71 		return;
72 
73 	spin_lock_irq(&b->irq_lock);
74 	if (b->irq_armed)
75 		__intel_breadcrumbs_disarm_irq(b);
76 	spin_unlock_irq(&b->irq_lock);
77 }
78 
79 static inline bool __request_completed(const struct i915_request *rq)
80 {
81 	return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
82 }
83 
84 __maybe_unused static bool
85 check_signal_order(struct intel_context *ce, struct i915_request *rq)
86 {
87 	if (!list_is_last(&rq->signal_link, &ce->signals) &&
88 	    i915_seqno_passed(rq->fence.seqno,
89 			      list_next_entry(rq, signal_link)->fence.seqno))
90 		return false;
91 
92 	if (!list_is_first(&rq->signal_link, &ce->signals) &&
93 	    i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
94 			      rq->fence.seqno))
95 		return false;
96 
97 	return true;
98 }
99 
100 static bool
101 __dma_fence_signal(struct dma_fence *fence)
102 {
103 	return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
104 }
105 
106 static void
107 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
108 {
109 	fence->timestamp = timestamp;
110 	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
111 	trace_dma_fence_signaled(fence);
112 }
113 
114 static void
115 __dma_fence_signal__notify(struct dma_fence *fence)
116 {
117 	struct dma_fence_cb *cur, *tmp;
118 
119 	lockdep_assert_held(fence->lock);
120 	lockdep_assert_irqs_disabled();
121 
122 	list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {
123 		INIT_LIST_HEAD(&cur->node);
124 		cur->func(fence, cur);
125 	}
126 	INIT_LIST_HEAD(&fence->cb_list);
127 }
128 
129 void intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine)
130 {
131 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
132 	const ktime_t timestamp = ktime_get();
133 	struct intel_context *ce, *cn;
134 	struct list_head *pos, *next;
135 	LIST_HEAD(signal);
136 
137 	spin_lock(&b->irq_lock);
138 
139 	if (b->irq_armed && list_empty(&b->signalers))
140 		__intel_breadcrumbs_disarm_irq(b);
141 
142 	list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) {
143 		GEM_BUG_ON(list_empty(&ce->signals));
144 
145 		list_for_each_safe(pos, next, &ce->signals) {
146 			struct i915_request *rq =
147 				list_entry(pos, typeof(*rq), signal_link);
148 
149 			GEM_BUG_ON(!check_signal_order(ce, rq));
150 
151 			if (!__request_completed(rq))
152 				break;
153 
154 			GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL,
155 					     &rq->fence.flags));
156 			clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
157 
158 			if (!__dma_fence_signal(&rq->fence))
159 				continue;
160 
161 			/*
162 			 * Queue for execution after dropping the signaling
163 			 * spinlock as the callback chain may end up adding
164 			 * more signalers to the same context or engine.
165 			 */
166 			i915_request_get(rq);
167 			list_add_tail(&rq->signal_link, &signal);
168 		}
169 
170 		/*
171 		 * We process the list deletion in bulk, only using a list_add
172 		 * (not list_move) above but keeping the status of
173 		 * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit.
174 		 */
175 		if (!list_is_first(pos, &ce->signals)) {
176 			/* Advance the list to the first incomplete request */
177 			__list_del_many(&ce->signals, pos);
178 			if (&ce->signals == pos) /* now empty */
179 				list_del_init(&ce->signal_link);
180 		}
181 	}
182 
183 	spin_unlock(&b->irq_lock);
184 
185 	list_for_each_safe(pos, next, &signal) {
186 		struct i915_request *rq =
187 			list_entry(pos, typeof(*rq), signal_link);
188 
189 		__dma_fence_signal__timestamp(&rq->fence, timestamp);
190 
191 		spin_lock(&rq->lock);
192 		__dma_fence_signal__notify(&rq->fence);
193 		spin_unlock(&rq->lock);
194 
195 		i915_request_put(rq);
196 	}
197 }
198 
199 void intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine)
200 {
201 	local_irq_disable();
202 	intel_engine_breadcrumbs_irq(engine);
203 	local_irq_enable();
204 }
205 
206 static void signal_irq_work(struct irq_work *work)
207 {
208 	struct intel_engine_cs *engine =
209 		container_of(work, typeof(*engine), breadcrumbs.irq_work);
210 
211 	intel_engine_breadcrumbs_irq(engine);
212 }
213 
214 void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine)
215 {
216 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
217 
218 	spin_lock_irq(&b->irq_lock);
219 	if (!b->irq_enabled++)
220 		irq_enable(engine);
221 	GEM_BUG_ON(!b->irq_enabled); /* no overflow! */
222 	spin_unlock_irq(&b->irq_lock);
223 }
224 
225 void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine)
226 {
227 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
228 
229 	spin_lock_irq(&b->irq_lock);
230 	GEM_BUG_ON(!b->irq_enabled); /* no underflow! */
231 	if (!--b->irq_enabled)
232 		irq_disable(engine);
233 	spin_unlock_irq(&b->irq_lock);
234 }
235 
236 static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
237 {
238 	struct intel_engine_cs *engine =
239 		container_of(b, struct intel_engine_cs, breadcrumbs);
240 
241 	lockdep_assert_held(&b->irq_lock);
242 	if (b->irq_armed)
243 		return;
244 
245 	/*
246 	 * The breadcrumb irq will be disarmed on the interrupt after the
247 	 * waiters are signaled. This gives us a single interrupt window in
248 	 * which we can add a new waiter and avoid the cost of re-enabling
249 	 * the irq.
250 	 */
251 	b->irq_armed = true;
252 
253 	/*
254 	 * Since we are waiting on a request, the GPU should be busy
255 	 * and should have its own rpm reference. This is tracked
256 	 * by i915->gt.awake, we can forgo holding our own wakref
257 	 * for the interrupt as before i915->gt.awake is released (when
258 	 * the driver is idle) we disarm the breadcrumbs.
259 	 */
260 
261 	if (!b->irq_enabled++)
262 		irq_enable(engine);
263 }
264 
265 void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
266 {
267 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
268 
269 	spin_lock_init(&b->irq_lock);
270 	INIT_LIST_HEAD(&b->signalers);
271 
272 	init_irq_work(&b->irq_work, signal_irq_work);
273 }
274 
275 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
276 {
277 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
278 	unsigned long flags;
279 
280 	spin_lock_irqsave(&b->irq_lock, flags);
281 
282 	if (b->irq_enabled)
283 		irq_enable(engine);
284 	else
285 		irq_disable(engine);
286 
287 	spin_unlock_irqrestore(&b->irq_lock, flags);
288 }
289 
290 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
291 {
292 }
293 
294 bool i915_request_enable_breadcrumb(struct i915_request *rq)
295 {
296 	lockdep_assert_held(&rq->lock);
297 	lockdep_assert_irqs_disabled();
298 
299 	if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
300 		struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
301 		struct intel_context *ce = rq->hw_context;
302 		struct list_head *pos;
303 
304 		spin_lock(&b->irq_lock);
305 		GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));
306 
307 		__intel_breadcrumbs_arm_irq(b);
308 
309 		/*
310 		 * We keep the seqno in retirement order, so we can break
311 		 * inside intel_engine_breadcrumbs_irq as soon as we've passed
312 		 * the last completed request (or seen a request that hasn't
313 		 * event started). We could iterate the timeline->requests list,
314 		 * but keeping a separate signalers_list has the advantage of
315 		 * hopefully being much smaller than the full list and so
316 		 * provides faster iteration and detection when there are no
317 		 * more interrupts required for this context.
318 		 *
319 		 * We typically expect to add new signalers in order, so we
320 		 * start looking for our insertion point from the tail of
321 		 * the list.
322 		 */
323 		list_for_each_prev(pos, &ce->signals) {
324 			struct i915_request *it =
325 				list_entry(pos, typeof(*it), signal_link);
326 
327 			if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
328 				break;
329 		}
330 		list_add(&rq->signal_link, pos);
331 		if (pos == &ce->signals) /* catch transitions from empty list */
332 			list_move_tail(&ce->signal_link, &b->signalers);
333 		GEM_BUG_ON(!check_signal_order(ce, rq));
334 
335 		set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
336 		spin_unlock(&b->irq_lock);
337 	}
338 
339 	return !__request_completed(rq);
340 }
341 
342 void i915_request_cancel_breadcrumb(struct i915_request *rq)
343 {
344 	struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
345 
346 	lockdep_assert_held(&rq->lock);
347 	lockdep_assert_irqs_disabled();
348 
349 	/*
350 	 * We must wait for b->irq_lock so that we know the interrupt handler
351 	 * has released its reference to the intel_context and has completed
352 	 * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if
353 	 * required).
354 	 */
355 	spin_lock(&b->irq_lock);
356 	if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
357 		struct intel_context *ce = rq->hw_context;
358 
359 		list_del(&rq->signal_link);
360 		if (list_empty(&ce->signals))
361 			list_del_init(&ce->signal_link);
362 
363 		clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
364 	}
365 	spin_unlock(&b->irq_lock);
366 }
367 
368 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
369 				    struct drm_printer *p)
370 {
371 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
372 	struct intel_context *ce;
373 	struct i915_request *rq;
374 
375 	if (list_empty(&b->signalers))
376 		return;
377 
378 	drm_printf(p, "Signals:\n");
379 
380 	spin_lock_irq(&b->irq_lock);
381 	list_for_each_entry(ce, &b->signalers, signal_link) {
382 		list_for_each_entry(rq, &ce->signals, signal_link) {
383 			drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
384 				   rq->fence.context, rq->fence.seqno,
385 				   i915_request_completed(rq) ? "!" :
386 				   i915_request_started(rq) ? "*" :
387 				   "",
388 				   jiffies_to_msecs(jiffies - rq->emitted_jiffies));
389 		}
390 	}
391 	spin_unlock_irq(&b->irq_lock);
392 }
393