1 /*
2  * Copyright © 2015 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/kthread.h>
26 #include <trace/events/dma_fence.h>
27 #include <uapi/linux/sched/types.h>
28 
29 #include "i915_drv.h"
30 #include "i915_trace.h"
31 
32 static void irq_enable(struct intel_engine_cs *engine)
33 {
34 	if (!engine->irq_enable)
35 		return;
36 
37 	/* Caller disables interrupts */
38 	spin_lock(&engine->gt->irq_lock);
39 	engine->irq_enable(engine);
40 	spin_unlock(&engine->gt->irq_lock);
41 }
42 
43 static void irq_disable(struct intel_engine_cs *engine)
44 {
45 	if (!engine->irq_disable)
46 		return;
47 
48 	/* Caller disables interrupts */
49 	spin_lock(&engine->gt->irq_lock);
50 	engine->irq_disable(engine);
51 	spin_unlock(&engine->gt->irq_lock);
52 }
53 
54 static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b)
55 {
56 	lockdep_assert_held(&b->irq_lock);
57 
58 	GEM_BUG_ON(!b->irq_enabled);
59 	if (!--b->irq_enabled)
60 		irq_disable(container_of(b,
61 					 struct intel_engine_cs,
62 					 breadcrumbs));
63 
64 	b->irq_armed = false;
65 }
66 
67 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
68 {
69 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
70 	unsigned long flags;
71 
72 	if (!b->irq_armed)
73 		return;
74 
75 	spin_lock_irqsave(&b->irq_lock, flags);
76 	if (b->irq_armed)
77 		__intel_breadcrumbs_disarm_irq(b);
78 	spin_unlock_irqrestore(&b->irq_lock, flags);
79 }
80 
81 static inline bool __request_completed(const struct i915_request *rq)
82 {
83 	return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
84 }
85 
86 __maybe_unused static bool
87 check_signal_order(struct intel_context *ce, struct i915_request *rq)
88 {
89 	if (!list_is_last(&rq->signal_link, &ce->signals) &&
90 	    i915_seqno_passed(rq->fence.seqno,
91 			      list_next_entry(rq, signal_link)->fence.seqno))
92 		return false;
93 
94 	if (!list_is_first(&rq->signal_link, &ce->signals) &&
95 	    i915_seqno_passed(list_prev_entry(rq, signal_link)->fence.seqno,
96 			      rq->fence.seqno))
97 		return false;
98 
99 	return true;
100 }
101 
102 static bool
103 __dma_fence_signal(struct dma_fence *fence)
104 {
105 	return !test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
106 }
107 
108 static void
109 __dma_fence_signal__timestamp(struct dma_fence *fence, ktime_t timestamp)
110 {
111 	fence->timestamp = timestamp;
112 	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
113 	trace_dma_fence_signaled(fence);
114 }
115 
116 static void
117 __dma_fence_signal__notify(struct dma_fence *fence,
118 			   const struct list_head *list)
119 {
120 	struct dma_fence_cb *cur, *tmp;
121 
122 	lockdep_assert_held(fence->lock);
123 	lockdep_assert_irqs_disabled();
124 
125 	list_for_each_entry_safe(cur, tmp, list, node) {
126 		INIT_LIST_HEAD(&cur->node);
127 		cur->func(fence, cur);
128 	}
129 }
130 
131 void intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine)
132 {
133 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
134 	const ktime_t timestamp = ktime_get();
135 	struct intel_context *ce, *cn;
136 	struct list_head *pos, *next;
137 	LIST_HEAD(signal);
138 
139 	spin_lock(&b->irq_lock);
140 
141 	if (b->irq_armed && list_empty(&b->signalers))
142 		__intel_breadcrumbs_disarm_irq(b);
143 
144 	list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) {
145 		GEM_BUG_ON(list_empty(&ce->signals));
146 
147 		list_for_each_safe(pos, next, &ce->signals) {
148 			struct i915_request *rq =
149 				list_entry(pos, typeof(*rq), signal_link);
150 
151 			GEM_BUG_ON(!check_signal_order(ce, rq));
152 
153 			if (!__request_completed(rq))
154 				break;
155 
156 			GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL,
157 					     &rq->fence.flags));
158 			clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
159 
160 			if (!__dma_fence_signal(&rq->fence))
161 				continue;
162 
163 			/*
164 			 * Queue for execution after dropping the signaling
165 			 * spinlock as the callback chain may end up adding
166 			 * more signalers to the same context or engine.
167 			 */
168 			i915_request_get(rq);
169 			list_add_tail(&rq->signal_link, &signal);
170 		}
171 
172 		/*
173 		 * We process the list deletion in bulk, only using a list_add
174 		 * (not list_move) above but keeping the status of
175 		 * rq->signal_link known with the I915_FENCE_FLAG_SIGNAL bit.
176 		 */
177 		if (!list_is_first(pos, &ce->signals)) {
178 			/* Advance the list to the first incomplete request */
179 			__list_del_many(&ce->signals, pos);
180 			if (&ce->signals == pos) /* now empty */
181 				list_del_init(&ce->signal_link);
182 		}
183 	}
184 
185 	spin_unlock(&b->irq_lock);
186 
187 	list_for_each_safe(pos, next, &signal) {
188 		struct i915_request *rq =
189 			list_entry(pos, typeof(*rq), signal_link);
190 		struct list_head cb_list;
191 
192 		spin_lock(&rq->lock);
193 		list_replace(&rq->fence.cb_list, &cb_list);
194 		__dma_fence_signal__timestamp(&rq->fence, timestamp);
195 		__dma_fence_signal__notify(&rq->fence, &cb_list);
196 		spin_unlock(&rq->lock);
197 
198 		i915_request_put(rq);
199 	}
200 }
201 
202 void intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine)
203 {
204 	local_irq_disable();
205 	intel_engine_breadcrumbs_irq(engine);
206 	local_irq_enable();
207 }
208 
209 static void signal_irq_work(struct irq_work *work)
210 {
211 	struct intel_engine_cs *engine =
212 		container_of(work, typeof(*engine), breadcrumbs.irq_work);
213 
214 	intel_engine_breadcrumbs_irq(engine);
215 }
216 
217 static void __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b)
218 {
219 	struct intel_engine_cs *engine =
220 		container_of(b, struct intel_engine_cs, breadcrumbs);
221 
222 	lockdep_assert_held(&b->irq_lock);
223 	if (b->irq_armed)
224 		return;
225 
226 	/*
227 	 * The breadcrumb irq will be disarmed on the interrupt after the
228 	 * waiters are signaled. This gives us a single interrupt window in
229 	 * which we can add a new waiter and avoid the cost of re-enabling
230 	 * the irq.
231 	 */
232 	b->irq_armed = true;
233 
234 	/*
235 	 * Since we are waiting on a request, the GPU should be busy
236 	 * and should have its own rpm reference. This is tracked
237 	 * by i915->gt.awake, we can forgo holding our own wakref
238 	 * for the interrupt as before i915->gt.awake is released (when
239 	 * the driver is idle) we disarm the breadcrumbs.
240 	 */
241 
242 	if (!b->irq_enabled++)
243 		irq_enable(engine);
244 }
245 
246 void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
247 {
248 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
249 
250 	spin_lock_init(&b->irq_lock);
251 	INIT_LIST_HEAD(&b->signalers);
252 
253 	init_irq_work(&b->irq_work, signal_irq_work);
254 }
255 
256 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine)
257 {
258 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
259 	unsigned long flags;
260 
261 	spin_lock_irqsave(&b->irq_lock, flags);
262 
263 	if (b->irq_enabled)
264 		irq_enable(engine);
265 	else
266 		irq_disable(engine);
267 
268 	spin_unlock_irqrestore(&b->irq_lock, flags);
269 }
270 
271 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
272 {
273 }
274 
275 bool i915_request_enable_breadcrumb(struct i915_request *rq)
276 {
277 	lockdep_assert_held(&rq->lock);
278 	lockdep_assert_irqs_disabled();
279 
280 	if (test_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags)) {
281 		struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
282 		struct intel_context *ce = rq->hw_context;
283 		struct list_head *pos;
284 
285 		spin_lock(&b->irq_lock);
286 		GEM_BUG_ON(test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags));
287 
288 		__intel_breadcrumbs_arm_irq(b);
289 
290 		/*
291 		 * We keep the seqno in retirement order, so we can break
292 		 * inside intel_engine_breadcrumbs_irq as soon as we've passed
293 		 * the last completed request (or seen a request that hasn't
294 		 * event started). We could iterate the timeline->requests list,
295 		 * but keeping a separate signalers_list has the advantage of
296 		 * hopefully being much smaller than the full list and so
297 		 * provides faster iteration and detection when there are no
298 		 * more interrupts required for this context.
299 		 *
300 		 * We typically expect to add new signalers in order, so we
301 		 * start looking for our insertion point from the tail of
302 		 * the list.
303 		 */
304 		list_for_each_prev(pos, &ce->signals) {
305 			struct i915_request *it =
306 				list_entry(pos, typeof(*it), signal_link);
307 
308 			if (i915_seqno_passed(rq->fence.seqno, it->fence.seqno))
309 				break;
310 		}
311 		list_add(&rq->signal_link, pos);
312 		if (pos == &ce->signals) /* catch transitions from empty list */
313 			list_move_tail(&ce->signal_link, &b->signalers);
314 		GEM_BUG_ON(!check_signal_order(ce, rq));
315 
316 		set_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
317 		spin_unlock(&b->irq_lock);
318 	}
319 
320 	return !__request_completed(rq);
321 }
322 
323 void i915_request_cancel_breadcrumb(struct i915_request *rq)
324 {
325 	struct intel_breadcrumbs *b = &rq->engine->breadcrumbs;
326 
327 	lockdep_assert_held(&rq->lock);
328 	lockdep_assert_irqs_disabled();
329 
330 	/*
331 	 * We must wait for b->irq_lock so that we know the interrupt handler
332 	 * has released its reference to the intel_context and has completed
333 	 * the DMA_FENCE_FLAG_SIGNALED_BIT/I915_FENCE_FLAG_SIGNAL dance (if
334 	 * required).
335 	 */
336 	spin_lock(&b->irq_lock);
337 	if (test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)) {
338 		struct intel_context *ce = rq->hw_context;
339 
340 		list_del(&rq->signal_link);
341 		if (list_empty(&ce->signals))
342 			list_del_init(&ce->signal_link);
343 
344 		clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags);
345 	}
346 	spin_unlock(&b->irq_lock);
347 }
348 
349 void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine,
350 				    struct drm_printer *p)
351 {
352 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
353 	struct intel_context *ce;
354 	struct i915_request *rq;
355 
356 	if (list_empty(&b->signalers))
357 		return;
358 
359 	drm_printf(p, "Signals:\n");
360 
361 	spin_lock_irq(&b->irq_lock);
362 	list_for_each_entry(ce, &b->signalers, signal_link) {
363 		list_for_each_entry(rq, &ce->signals, signal_link) {
364 			drm_printf(p, "\t[%llx:%llx%s] @ %dms\n",
365 				   rq->fence.context, rq->fence.seqno,
366 				   i915_request_completed(rq) ? "!" :
367 				   i915_request_started(rq) ? "*" :
368 				   "",
369 				   jiffies_to_msecs(jiffies - rq->emitted_jiffies));
370 		}
371 	}
372 	spin_unlock_irq(&b->irq_lock);
373 }
374