1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2016-2018 Intel Corporation
5  */
6 
7 #include "i915_drv.h"
8 
9 #include "i915_active.h"
10 #include "i915_syncmap.h"
11 #include "intel_gt.h"
12 #include "intel_ring.h"
13 #include "intel_timeline.h"
14 
15 #define ptr_set_bit(ptr, bit) ((typeof(ptr))((unsigned long)(ptr) | BIT(bit)))
16 #define ptr_test_bit(ptr, bit) ((unsigned long)(ptr) & BIT(bit))
17 
18 #define CACHELINE_BITS 6
19 #define CACHELINE_FREE CACHELINE_BITS
20 
21 struct intel_timeline_hwsp {
22 	struct intel_gt *gt;
23 	struct intel_gt_timelines *gt_timelines;
24 	struct list_head free_link;
25 	struct i915_vma *vma;
26 	u64 free_bitmap;
27 };
28 
29 static struct i915_vma *__hwsp_alloc(struct intel_gt *gt)
30 {
31 	struct drm_i915_private *i915 = gt->i915;
32 	struct drm_i915_gem_object *obj;
33 	struct i915_vma *vma;
34 
35 	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
36 	if (IS_ERR(obj))
37 		return ERR_CAST(obj);
38 
39 	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
40 
41 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
42 	if (IS_ERR(vma))
43 		i915_gem_object_put(obj);
44 
45 	return vma;
46 }
47 
48 static struct i915_vma *
49 hwsp_alloc(struct intel_timeline *timeline, unsigned int *cacheline)
50 {
51 	struct intel_gt_timelines *gt = &timeline->gt->timelines;
52 	struct intel_timeline_hwsp *hwsp;
53 
54 	BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
55 
56 	spin_lock_irq(&gt->hwsp_lock);
57 
58 	/* hwsp_free_list only contains HWSP that have available cachelines */
59 	hwsp = list_first_entry_or_null(&gt->hwsp_free_list,
60 					typeof(*hwsp), free_link);
61 	if (!hwsp) {
62 		struct i915_vma *vma;
63 
64 		spin_unlock_irq(&gt->hwsp_lock);
65 
66 		hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
67 		if (!hwsp)
68 			return ERR_PTR(-ENOMEM);
69 
70 		vma = __hwsp_alloc(timeline->gt);
71 		if (IS_ERR(vma)) {
72 			kfree(hwsp);
73 			return vma;
74 		}
75 
76 		GT_TRACE(timeline->gt, "new HWSP allocated\n");
77 
78 		vma->private = hwsp;
79 		hwsp->gt = timeline->gt;
80 		hwsp->vma = vma;
81 		hwsp->free_bitmap = ~0ull;
82 		hwsp->gt_timelines = gt;
83 
84 		spin_lock_irq(&gt->hwsp_lock);
85 		list_add(&hwsp->free_link, &gt->hwsp_free_list);
86 	}
87 
88 	GEM_BUG_ON(!hwsp->free_bitmap);
89 	*cacheline = __ffs64(hwsp->free_bitmap);
90 	hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
91 	if (!hwsp->free_bitmap)
92 		list_del(&hwsp->free_link);
93 
94 	spin_unlock_irq(&gt->hwsp_lock);
95 
96 	GEM_BUG_ON(hwsp->vma->private != hwsp);
97 	return hwsp->vma;
98 }
99 
100 static void __idle_hwsp_free(struct intel_timeline_hwsp *hwsp, int cacheline)
101 {
102 	struct intel_gt_timelines *gt = hwsp->gt_timelines;
103 	unsigned long flags;
104 
105 	spin_lock_irqsave(&gt->hwsp_lock, flags);
106 
107 	/* As a cacheline becomes available, publish the HWSP on the freelist */
108 	if (!hwsp->free_bitmap)
109 		list_add_tail(&hwsp->free_link, &gt->hwsp_free_list);
110 
111 	GEM_BUG_ON(cacheline >= BITS_PER_TYPE(hwsp->free_bitmap));
112 	hwsp->free_bitmap |= BIT_ULL(cacheline);
113 
114 	/* And if no one is left using it, give the page back to the system */
115 	if (hwsp->free_bitmap == ~0ull) {
116 		i915_vma_put(hwsp->vma);
117 		list_del(&hwsp->free_link);
118 		kfree(hwsp);
119 	}
120 
121 	spin_unlock_irqrestore(&gt->hwsp_lock, flags);
122 }
123 
124 static void __rcu_cacheline_free(struct rcu_head *rcu)
125 {
126 	struct intel_timeline_cacheline *cl =
127 		container_of(rcu, typeof(*cl), rcu);
128 
129 	/* Must wait until after all *rq->hwsp are complete before removing */
130 	i915_gem_object_unpin_map(cl->hwsp->vma->obj);
131 	__idle_hwsp_free(cl->hwsp, ptr_unmask_bits(cl->vaddr, CACHELINE_BITS));
132 
133 	i915_active_fini(&cl->active);
134 	kfree(cl);
135 }
136 
137 static void __idle_cacheline_free(struct intel_timeline_cacheline *cl)
138 {
139 	GEM_BUG_ON(!i915_active_is_idle(&cl->active));
140 	call_rcu(&cl->rcu, __rcu_cacheline_free);
141 }
142 
143 __i915_active_call
144 static void __cacheline_retire(struct i915_active *active)
145 {
146 	struct intel_timeline_cacheline *cl =
147 		container_of(active, typeof(*cl), active);
148 
149 	i915_vma_unpin(cl->hwsp->vma);
150 	if (ptr_test_bit(cl->vaddr, CACHELINE_FREE))
151 		__idle_cacheline_free(cl);
152 }
153 
154 static int __cacheline_active(struct i915_active *active)
155 {
156 	struct intel_timeline_cacheline *cl =
157 		container_of(active, typeof(*cl), active);
158 
159 	__i915_vma_pin(cl->hwsp->vma);
160 	return 0;
161 }
162 
163 static struct intel_timeline_cacheline *
164 cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline)
165 {
166 	struct intel_timeline_cacheline *cl;
167 	void *vaddr;
168 
169 	GEM_BUG_ON(cacheline >= BIT(CACHELINE_BITS));
170 
171 	cl = kmalloc(sizeof(*cl), GFP_KERNEL);
172 	if (!cl)
173 		return ERR_PTR(-ENOMEM);
174 
175 	vaddr = i915_gem_object_pin_map(hwsp->vma->obj, I915_MAP_WB);
176 	if (IS_ERR(vaddr)) {
177 		kfree(cl);
178 		return ERR_CAST(vaddr);
179 	}
180 
181 	cl->hwsp = hwsp;
182 	cl->vaddr = page_pack_bits(vaddr, cacheline);
183 
184 	i915_active_init(&cl->active, __cacheline_active, __cacheline_retire);
185 
186 	return cl;
187 }
188 
189 static void cacheline_acquire(struct intel_timeline_cacheline *cl,
190 			      u32 ggtt_offset)
191 {
192 	if (!cl)
193 		return;
194 
195 	cl->ggtt_offset = ggtt_offset;
196 	i915_active_acquire(&cl->active);
197 }
198 
199 static void cacheline_release(struct intel_timeline_cacheline *cl)
200 {
201 	if (cl)
202 		i915_active_release(&cl->active);
203 }
204 
205 static void cacheline_free(struct intel_timeline_cacheline *cl)
206 {
207 	if (!i915_active_acquire_if_busy(&cl->active)) {
208 		__idle_cacheline_free(cl);
209 		return;
210 	}
211 
212 	GEM_BUG_ON(ptr_test_bit(cl->vaddr, CACHELINE_FREE));
213 	cl->vaddr = ptr_set_bit(cl->vaddr, CACHELINE_FREE);
214 
215 	i915_active_release(&cl->active);
216 }
217 
218 static int intel_timeline_init(struct intel_timeline *timeline,
219 			       struct intel_gt *gt,
220 			       struct i915_vma *hwsp,
221 			       unsigned int offset)
222 {
223 	void *vaddr;
224 
225 	kref_init(&timeline->kref);
226 	atomic_set(&timeline->pin_count, 0);
227 
228 	timeline->gt = gt;
229 
230 	timeline->has_initial_breadcrumb = !hwsp;
231 	timeline->hwsp_cacheline = NULL;
232 
233 	if (!hwsp) {
234 		struct intel_timeline_cacheline *cl;
235 		unsigned int cacheline;
236 
237 		hwsp = hwsp_alloc(timeline, &cacheline);
238 		if (IS_ERR(hwsp))
239 			return PTR_ERR(hwsp);
240 
241 		cl = cacheline_alloc(hwsp->private, cacheline);
242 		if (IS_ERR(cl)) {
243 			__idle_hwsp_free(hwsp->private, cacheline);
244 			return PTR_ERR(cl);
245 		}
246 
247 		timeline->hwsp_cacheline = cl;
248 		timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
249 
250 		vaddr = page_mask_bits(cl->vaddr);
251 	} else {
252 		timeline->hwsp_offset = offset;
253 		vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
254 		if (IS_ERR(vaddr))
255 			return PTR_ERR(vaddr);
256 	}
257 
258 	timeline->hwsp_seqno =
259 		memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);
260 
261 	timeline->hwsp_ggtt = i915_vma_get(hwsp);
262 	GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);
263 
264 	timeline->fence_context = dma_fence_context_alloc(1);
265 
266 	mutex_init(&timeline->mutex);
267 
268 	INIT_ACTIVE_FENCE(&timeline->last_request);
269 	INIT_LIST_HEAD(&timeline->requests);
270 
271 	i915_syncmap_init(&timeline->sync);
272 
273 	return 0;
274 }
275 
276 void intel_gt_init_timelines(struct intel_gt *gt)
277 {
278 	struct intel_gt_timelines *timelines = &gt->timelines;
279 
280 	spin_lock_init(&timelines->lock);
281 	INIT_LIST_HEAD(&timelines->active_list);
282 
283 	spin_lock_init(&timelines->hwsp_lock);
284 	INIT_LIST_HEAD(&timelines->hwsp_free_list);
285 }
286 
287 static void intel_timeline_fini(struct intel_timeline *timeline)
288 {
289 	GEM_BUG_ON(atomic_read(&timeline->pin_count));
290 	GEM_BUG_ON(!list_empty(&timeline->requests));
291 	GEM_BUG_ON(timeline->retire);
292 
293 	if (timeline->hwsp_cacheline)
294 		cacheline_free(timeline->hwsp_cacheline);
295 	else
296 		i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
297 
298 	i915_vma_put(timeline->hwsp_ggtt);
299 }
300 
301 struct intel_timeline *
302 __intel_timeline_create(struct intel_gt *gt,
303 			struct i915_vma *global_hwsp,
304 			unsigned int offset)
305 {
306 	struct intel_timeline *timeline;
307 	int err;
308 
309 	timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
310 	if (!timeline)
311 		return ERR_PTR(-ENOMEM);
312 
313 	err = intel_timeline_init(timeline, gt, global_hwsp, offset);
314 	if (err) {
315 		kfree(timeline);
316 		return ERR_PTR(err);
317 	}
318 
319 	return timeline;
320 }
321 
322 struct intel_timeline *
323 intel_timeline_create_from_engine(struct intel_engine_cs *engine,
324 				  unsigned int offset)
325 {
326 	struct i915_vma *hwsp = engine->status_page.vma;
327 	struct intel_timeline *tl;
328 
329 	tl = __intel_timeline_create(engine->gt, hwsp, offset);
330 	if (IS_ERR(tl))
331 		return tl;
332 
333 	/* Borrow a nearby lock; we only create these timelines during init */
334 	mutex_lock(&hwsp->vm->mutex);
335 	list_add_tail(&tl->engine_link, &engine->status_page.timelines);
336 	mutex_unlock(&hwsp->vm->mutex);
337 
338 	return tl;
339 }
340 
341 void __intel_timeline_pin(struct intel_timeline *tl)
342 {
343 	GEM_BUG_ON(!atomic_read(&tl->pin_count));
344 	atomic_inc(&tl->pin_count);
345 }
346 
347 int intel_timeline_pin(struct intel_timeline *tl, struct i915_gem_ww_ctx *ww)
348 {
349 	int err;
350 
351 	if (atomic_add_unless(&tl->pin_count, 1, 0))
352 		return 0;
353 
354 	err = i915_ggtt_pin(tl->hwsp_ggtt, ww, 0, PIN_HIGH);
355 	if (err)
356 		return err;
357 
358 	tl->hwsp_offset =
359 		i915_ggtt_offset(tl->hwsp_ggtt) +
360 		offset_in_page(tl->hwsp_offset);
361 	GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
362 		 tl->fence_context, tl->hwsp_offset);
363 
364 	cacheline_acquire(tl->hwsp_cacheline, tl->hwsp_offset);
365 	if (atomic_fetch_inc(&tl->pin_count)) {
366 		cacheline_release(tl->hwsp_cacheline);
367 		__i915_vma_unpin(tl->hwsp_ggtt);
368 	}
369 
370 	return 0;
371 }
372 
373 void intel_timeline_reset_seqno(const struct intel_timeline *tl)
374 {
375 	/* Must be pinned to be writable, and no requests in flight. */
376 	GEM_BUG_ON(!atomic_read(&tl->pin_count));
377 	WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);
378 }
379 
380 void intel_timeline_enter(struct intel_timeline *tl)
381 {
382 	struct intel_gt_timelines *timelines = &tl->gt->timelines;
383 
384 	/*
385 	 * Pretend we are serialised by the timeline->mutex.
386 	 *
387 	 * While generally true, there are a few exceptions to the rule
388 	 * for the engine->kernel_context being used to manage power
389 	 * transitions. As the engine_park may be called from under any
390 	 * timeline, it uses the power mutex as a global serialisation
391 	 * lock to prevent any other request entering its timeline.
392 	 *
393 	 * The rule is generally tl->mutex, otherwise engine->wakeref.mutex.
394 	 *
395 	 * However, intel_gt_retire_request() does not know which engine
396 	 * it is retiring along and so cannot partake in the engine-pm
397 	 * barrier, and there we use the tl->active_count as a means to
398 	 * pin the timeline in the active_list while the locks are dropped.
399 	 * Ergo, as that is outside of the engine-pm barrier, we need to
400 	 * use atomic to manipulate tl->active_count.
401 	 */
402 	lockdep_assert_held(&tl->mutex);
403 
404 	if (atomic_add_unless(&tl->active_count, 1, 0))
405 		return;
406 
407 	spin_lock(&timelines->lock);
408 	if (!atomic_fetch_inc(&tl->active_count)) {
409 		/*
410 		 * The HWSP is volatile, and may have been lost while inactive,
411 		 * e.g. across suspend/resume. Be paranoid, and ensure that
412 		 * the HWSP value matches our seqno so we don't proclaim
413 		 * the next request as already complete.
414 		 */
415 		intel_timeline_reset_seqno(tl);
416 		list_add_tail(&tl->link, &timelines->active_list);
417 	}
418 	spin_unlock(&timelines->lock);
419 }
420 
421 void intel_timeline_exit(struct intel_timeline *tl)
422 {
423 	struct intel_gt_timelines *timelines = &tl->gt->timelines;
424 
425 	/* See intel_timeline_enter() */
426 	lockdep_assert_held(&tl->mutex);
427 
428 	GEM_BUG_ON(!atomic_read(&tl->active_count));
429 	if (atomic_add_unless(&tl->active_count, -1, 1))
430 		return;
431 
432 	spin_lock(&timelines->lock);
433 	if (atomic_dec_and_test(&tl->active_count))
434 		list_del(&tl->link);
435 	spin_unlock(&timelines->lock);
436 
437 	/*
438 	 * Since this timeline is idle, all bariers upon which we were waiting
439 	 * must also be complete and so we can discard the last used barriers
440 	 * without loss of information.
441 	 */
442 	i915_syncmap_free(&tl->sync);
443 }
444 
445 static u32 timeline_advance(struct intel_timeline *tl)
446 {
447 	GEM_BUG_ON(!atomic_read(&tl->pin_count));
448 	GEM_BUG_ON(tl->seqno & tl->has_initial_breadcrumb);
449 
450 	return tl->seqno += 1 + tl->has_initial_breadcrumb;
451 }
452 
453 static void timeline_rollback(struct intel_timeline *tl)
454 {
455 	tl->seqno -= 1 + tl->has_initial_breadcrumb;
456 }
457 
458 static noinline int
459 __intel_timeline_get_seqno(struct intel_timeline *tl,
460 			   struct i915_request *rq,
461 			   u32 *seqno)
462 {
463 	struct intel_timeline_cacheline *cl;
464 	unsigned int cacheline;
465 	struct i915_vma *vma;
466 	void *vaddr;
467 	int err;
468 
469 	might_lock(&tl->gt->ggtt->vm.mutex);
470 	GT_TRACE(tl->gt, "timeline:%llx wrapped\n", tl->fence_context);
471 
472 	/*
473 	 * If there is an outstanding GPU reference to this cacheline,
474 	 * such as it being sampled by a HW semaphore on another timeline,
475 	 * we cannot wraparound our seqno value (the HW semaphore does
476 	 * a strict greater-than-or-equals compare, not i915_seqno_passed).
477 	 * So if the cacheline is still busy, we must detach ourselves
478 	 * from it and leave it inflight alongside its users.
479 	 *
480 	 * However, if nobody is watching and we can guarantee that nobody
481 	 * will, we could simply reuse the same cacheline.
482 	 *
483 	 * if (i915_active_request_is_signaled(&tl->last_request) &&
484 	 *     i915_active_is_signaled(&tl->hwsp_cacheline->active))
485 	 *	return 0;
486 	 *
487 	 * That seems unlikely for a busy timeline that needed to wrap in
488 	 * the first place, so just replace the cacheline.
489 	 */
490 
491 	vma = hwsp_alloc(tl, &cacheline);
492 	if (IS_ERR(vma)) {
493 		err = PTR_ERR(vma);
494 		goto err_rollback;
495 	}
496 
497 	err = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);
498 	if (err) {
499 		__idle_hwsp_free(vma->private, cacheline);
500 		goto err_rollback;
501 	}
502 
503 	cl = cacheline_alloc(vma->private, cacheline);
504 	if (IS_ERR(cl)) {
505 		err = PTR_ERR(cl);
506 		__idle_hwsp_free(vma->private, cacheline);
507 		goto err_unpin;
508 	}
509 	GEM_BUG_ON(cl->hwsp->vma != vma);
510 
511 	/*
512 	 * Attach the old cacheline to the current request, so that we only
513 	 * free it after the current request is retired, which ensures that
514 	 * all writes into the cacheline from previous requests are complete.
515 	 */
516 	err = i915_active_ref(&tl->hwsp_cacheline->active,
517 			      tl->fence_context,
518 			      &rq->fence);
519 	if (err)
520 		goto err_cacheline;
521 
522 	cacheline_release(tl->hwsp_cacheline); /* ownership now xfered to rq */
523 	cacheline_free(tl->hwsp_cacheline);
524 
525 	i915_vma_unpin(tl->hwsp_ggtt); /* binding kept alive by old cacheline */
526 	i915_vma_put(tl->hwsp_ggtt);
527 
528 	tl->hwsp_ggtt = i915_vma_get(vma);
529 
530 	vaddr = page_mask_bits(cl->vaddr);
531 	tl->hwsp_offset = cacheline * CACHELINE_BYTES;
532 	tl->hwsp_seqno =
533 		memset(vaddr + tl->hwsp_offset, 0, CACHELINE_BYTES);
534 
535 	tl->hwsp_offset += i915_ggtt_offset(vma);
536 	GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
537 		 tl->fence_context, tl->hwsp_offset);
538 
539 	cacheline_acquire(cl, tl->hwsp_offset);
540 	tl->hwsp_cacheline = cl;
541 
542 	*seqno = timeline_advance(tl);
543 	GEM_BUG_ON(i915_seqno_passed(*tl->hwsp_seqno, *seqno));
544 	return 0;
545 
546 err_cacheline:
547 	cacheline_free(cl);
548 err_unpin:
549 	i915_vma_unpin(vma);
550 err_rollback:
551 	timeline_rollback(tl);
552 	return err;
553 }
554 
555 int intel_timeline_get_seqno(struct intel_timeline *tl,
556 			     struct i915_request *rq,
557 			     u32 *seqno)
558 {
559 	*seqno = timeline_advance(tl);
560 
561 	/* Replace the HWSP on wraparound for HW semaphores */
562 	if (unlikely(!*seqno && tl->hwsp_cacheline))
563 		return __intel_timeline_get_seqno(tl, rq, seqno);
564 
565 	return 0;
566 }
567 
568 static int cacheline_ref(struct intel_timeline_cacheline *cl,
569 			 struct i915_request *rq)
570 {
571 	return i915_active_add_request(&cl->active, rq);
572 }
573 
574 int intel_timeline_read_hwsp(struct i915_request *from,
575 			     struct i915_request *to,
576 			     u32 *hwsp)
577 {
578 	struct intel_timeline_cacheline *cl;
579 	int err;
580 
581 	GEM_BUG_ON(!rcu_access_pointer(from->hwsp_cacheline));
582 
583 	rcu_read_lock();
584 	cl = rcu_dereference(from->hwsp_cacheline);
585 	if (i915_request_signaled(from)) /* confirm cacheline is valid */
586 		goto unlock;
587 	if (unlikely(!i915_active_acquire_if_busy(&cl->active)))
588 		goto unlock; /* seqno wrapped and completed! */
589 	if (unlikely(__i915_request_is_complete(from)))
590 		goto release;
591 	rcu_read_unlock();
592 
593 	err = cacheline_ref(cl, to);
594 	if (err)
595 		goto out;
596 
597 	*hwsp = cl->ggtt_offset;
598 out:
599 	i915_active_release(&cl->active);
600 	return err;
601 
602 release:
603 	i915_active_release(&cl->active);
604 unlock:
605 	rcu_read_unlock();
606 	return 1;
607 }
608 
609 void intel_timeline_unpin(struct intel_timeline *tl)
610 {
611 	GEM_BUG_ON(!atomic_read(&tl->pin_count));
612 	if (!atomic_dec_and_test(&tl->pin_count))
613 		return;
614 
615 	cacheline_release(tl->hwsp_cacheline);
616 
617 	__i915_vma_unpin(tl->hwsp_ggtt);
618 }
619 
620 void __intel_timeline_free(struct kref *kref)
621 {
622 	struct intel_timeline *timeline =
623 		container_of(kref, typeof(*timeline), kref);
624 
625 	intel_timeline_fini(timeline);
626 	kfree_rcu(timeline, rcu);
627 }
628 
629 void intel_gt_fini_timelines(struct intel_gt *gt)
630 {
631 	struct intel_gt_timelines *timelines = &gt->timelines;
632 
633 	GEM_BUG_ON(!list_empty(&timelines->active_list));
634 	GEM_BUG_ON(!list_empty(&timelines->hwsp_free_list));
635 }
636 
637 void intel_gt_show_timelines(struct intel_gt *gt,
638 			     struct drm_printer *m,
639 			     void (*show_request)(struct drm_printer *m,
640 						  const struct i915_request *rq,
641 						  const char *prefix,
642 						  int indent))
643 {
644 	struct intel_gt_timelines *timelines = &gt->timelines;
645 	struct intel_timeline *tl, *tn;
646 	LIST_HEAD(free);
647 
648 	spin_lock(&timelines->lock);
649 	list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {
650 		unsigned long count, ready, inflight;
651 		struct i915_request *rq, *rn;
652 		struct dma_fence *fence;
653 
654 		if (!mutex_trylock(&tl->mutex)) {
655 			drm_printf(m, "Timeline %llx: busy; skipping\n",
656 				   tl->fence_context);
657 			continue;
658 		}
659 
660 		intel_timeline_get(tl);
661 		GEM_BUG_ON(!atomic_read(&tl->active_count));
662 		atomic_inc(&tl->active_count); /* pin the list element */
663 		spin_unlock(&timelines->lock);
664 
665 		count = 0;
666 		ready = 0;
667 		inflight = 0;
668 		list_for_each_entry_safe(rq, rn, &tl->requests, link) {
669 			if (i915_request_completed(rq))
670 				continue;
671 
672 			count++;
673 			if (i915_request_is_ready(rq))
674 				ready++;
675 			if (i915_request_is_active(rq))
676 				inflight++;
677 		}
678 
679 		drm_printf(m, "Timeline %llx: { ", tl->fence_context);
680 		drm_printf(m, "count: %lu, ready: %lu, inflight: %lu",
681 			   count, ready, inflight);
682 		drm_printf(m, ", seqno: { current: %d, last: %d }",
683 			   *tl->hwsp_seqno, tl->seqno);
684 		fence = i915_active_fence_get(&tl->last_request);
685 		if (fence) {
686 			drm_printf(m, ", engine: %s",
687 				   to_request(fence)->engine->name);
688 			dma_fence_put(fence);
689 		}
690 		drm_printf(m, " }\n");
691 
692 		if (show_request) {
693 			list_for_each_entry_safe(rq, rn, &tl->requests, link)
694 				show_request(m, rq, "", 2);
695 		}
696 
697 		mutex_unlock(&tl->mutex);
698 		spin_lock(&timelines->lock);
699 
700 		/* Resume list iteration after reacquiring spinlock */
701 		list_safe_reset_next(tl, tn, link);
702 		if (atomic_dec_and_test(&tl->active_count))
703 			list_del(&tl->link);
704 
705 		/* Defer the final release to after the spinlock */
706 		if (refcount_dec_and_test(&tl->kref.refcount)) {
707 			GEM_BUG_ON(atomic_read(&tl->active_count));
708 			list_add(&tl->link, &free);
709 		}
710 	}
711 	spin_unlock(&timelines->lock);
712 
713 	list_for_each_entry_safe(tl, tn, &free, link)
714 		__intel_timeline_free(&tl->kref);
715 }
716 
717 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
718 #include "gt/selftests/mock_timeline.c"
719 #include "gt/selftest_timeline.c"
720 #endif
721