1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2016-2018 Intel Corporation
4  */
5 
6 #include "i915_drv.h"
7 
8 #include "i915_active.h"
9 #include "i915_syncmap.h"
10 #include "intel_gt.h"
11 #include "intel_ring.h"
12 #include "intel_timeline.h"
13 
14 #define TIMELINE_SEQNO_BYTES 8
15 
16 static struct i915_vma *hwsp_alloc(struct intel_gt *gt)
17 {
18 	struct drm_i915_private *i915 = gt->i915;
19 	struct drm_i915_gem_object *obj;
20 	struct i915_vma *vma;
21 
22 	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
23 	if (IS_ERR(obj))
24 		return ERR_CAST(obj);
25 
26 	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
27 
28 	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
29 	if (IS_ERR(vma))
30 		i915_gem_object_put(obj);
31 
32 	return vma;
33 }
34 
35 static void __timeline_retire(struct i915_active *active)
36 {
37 	struct intel_timeline *tl =
38 		container_of(active, typeof(*tl), active);
39 
40 	i915_vma_unpin(tl->hwsp_ggtt);
41 	intel_timeline_put(tl);
42 }
43 
44 static int __timeline_active(struct i915_active *active)
45 {
46 	struct intel_timeline *tl =
47 		container_of(active, typeof(*tl), active);
48 
49 	__i915_vma_pin(tl->hwsp_ggtt);
50 	intel_timeline_get(tl);
51 	return 0;
52 }
53 
54 I915_SELFTEST_EXPORT int
55 intel_timeline_pin_map(struct intel_timeline *timeline)
56 {
57 	struct drm_i915_gem_object *obj = timeline->hwsp_ggtt->obj;
58 	u32 ofs = offset_in_page(timeline->hwsp_offset);
59 	void *vaddr;
60 
61 	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
62 	if (IS_ERR(vaddr))
63 		return PTR_ERR(vaddr);
64 
65 	timeline->hwsp_map = vaddr;
66 	timeline->hwsp_seqno = memset(vaddr + ofs, 0, TIMELINE_SEQNO_BYTES);
67 	clflush(vaddr + ofs);
68 
69 	return 0;
70 }
71 
72 static int intel_timeline_init(struct intel_timeline *timeline,
73 			       struct intel_gt *gt,
74 			       struct i915_vma *hwsp,
75 			       unsigned int offset)
76 {
77 	kref_init(&timeline->kref);
78 	atomic_set(&timeline->pin_count, 0);
79 
80 	timeline->gt = gt;
81 
82 	if (hwsp) {
83 		timeline->hwsp_offset = offset;
84 		timeline->hwsp_ggtt = i915_vma_get(hwsp);
85 	} else {
86 		timeline->has_initial_breadcrumb = true;
87 		hwsp = hwsp_alloc(gt);
88 		if (IS_ERR(hwsp))
89 			return PTR_ERR(hwsp);
90 		timeline->hwsp_ggtt = hwsp;
91 	}
92 
93 	timeline->hwsp_map = NULL;
94 	timeline->hwsp_seqno = (void *)(long)timeline->hwsp_offset;
95 
96 	GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);
97 
98 	timeline->fence_context = dma_fence_context_alloc(1);
99 
100 	mutex_init(&timeline->mutex);
101 
102 	INIT_ACTIVE_FENCE(&timeline->last_request);
103 	INIT_LIST_HEAD(&timeline->requests);
104 
105 	i915_syncmap_init(&timeline->sync);
106 	i915_active_init(&timeline->active, __timeline_active,
107 			 __timeline_retire, 0);
108 
109 	return 0;
110 }
111 
112 void intel_gt_init_timelines(struct intel_gt *gt)
113 {
114 	struct intel_gt_timelines *timelines = &gt->timelines;
115 
116 	spin_lock_init(&timelines->lock);
117 	INIT_LIST_HEAD(&timelines->active_list);
118 }
119 
120 static void intel_timeline_fini(struct rcu_head *rcu)
121 {
122 	struct intel_timeline *timeline =
123 		container_of(rcu, struct intel_timeline, rcu);
124 
125 	if (timeline->hwsp_map)
126 		i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
127 
128 	i915_vma_put(timeline->hwsp_ggtt);
129 	i915_active_fini(&timeline->active);
130 	kfree(timeline);
131 }
132 
133 struct intel_timeline *
134 __intel_timeline_create(struct intel_gt *gt,
135 			struct i915_vma *global_hwsp,
136 			unsigned int offset)
137 {
138 	struct intel_timeline *timeline;
139 	int err;
140 
141 	timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
142 	if (!timeline)
143 		return ERR_PTR(-ENOMEM);
144 
145 	err = intel_timeline_init(timeline, gt, global_hwsp, offset);
146 	if (err) {
147 		kfree(timeline);
148 		return ERR_PTR(err);
149 	}
150 
151 	return timeline;
152 }
153 
154 struct intel_timeline *
155 intel_timeline_create_from_engine(struct intel_engine_cs *engine,
156 				  unsigned int offset)
157 {
158 	struct i915_vma *hwsp = engine->status_page.vma;
159 	struct intel_timeline *tl;
160 
161 	tl = __intel_timeline_create(engine->gt, hwsp, offset);
162 	if (IS_ERR(tl))
163 		return tl;
164 
165 	/* Borrow a nearby lock; we only create these timelines during init */
166 	mutex_lock(&hwsp->vm->mutex);
167 	list_add_tail(&tl->engine_link, &engine->status_page.timelines);
168 	mutex_unlock(&hwsp->vm->mutex);
169 
170 	return tl;
171 }
172 
173 void __intel_timeline_pin(struct intel_timeline *tl)
174 {
175 	GEM_BUG_ON(!atomic_read(&tl->pin_count));
176 	atomic_inc(&tl->pin_count);
177 }
178 
179 int intel_timeline_pin(struct intel_timeline *tl, struct i915_gem_ww_ctx *ww)
180 {
181 	int err;
182 
183 	if (atomic_add_unless(&tl->pin_count, 1, 0))
184 		return 0;
185 
186 	if (!tl->hwsp_map) {
187 		err = intel_timeline_pin_map(tl);
188 		if (err)
189 			return err;
190 	}
191 
192 	err = i915_ggtt_pin(tl->hwsp_ggtt, ww, 0, PIN_HIGH);
193 	if (err)
194 		return err;
195 
196 	tl->hwsp_offset =
197 		i915_ggtt_offset(tl->hwsp_ggtt) +
198 		offset_in_page(tl->hwsp_offset);
199 	GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
200 		 tl->fence_context, tl->hwsp_offset);
201 
202 	i915_active_acquire(&tl->active);
203 	if (atomic_fetch_inc(&tl->pin_count)) {
204 		i915_active_release(&tl->active);
205 		__i915_vma_unpin(tl->hwsp_ggtt);
206 	}
207 
208 	return 0;
209 }
210 
211 void intel_timeline_reset_seqno(const struct intel_timeline *tl)
212 {
213 	u32 *hwsp_seqno = (u32 *)tl->hwsp_seqno;
214 	/* Must be pinned to be writable, and no requests in flight. */
215 	GEM_BUG_ON(!atomic_read(&tl->pin_count));
216 
217 	memset(hwsp_seqno + 1, 0, TIMELINE_SEQNO_BYTES - sizeof(*hwsp_seqno));
218 	WRITE_ONCE(*hwsp_seqno, tl->seqno);
219 	clflush(hwsp_seqno);
220 }
221 
222 void intel_timeline_enter(struct intel_timeline *tl)
223 {
224 	struct intel_gt_timelines *timelines = &tl->gt->timelines;
225 
226 	/*
227 	 * Pretend we are serialised by the timeline->mutex.
228 	 *
229 	 * While generally true, there are a few exceptions to the rule
230 	 * for the engine->kernel_context being used to manage power
231 	 * transitions. As the engine_park may be called from under any
232 	 * timeline, it uses the power mutex as a global serialisation
233 	 * lock to prevent any other request entering its timeline.
234 	 *
235 	 * The rule is generally tl->mutex, otherwise engine->wakeref.mutex.
236 	 *
237 	 * However, intel_gt_retire_request() does not know which engine
238 	 * it is retiring along and so cannot partake in the engine-pm
239 	 * barrier, and there we use the tl->active_count as a means to
240 	 * pin the timeline in the active_list while the locks are dropped.
241 	 * Ergo, as that is outside of the engine-pm barrier, we need to
242 	 * use atomic to manipulate tl->active_count.
243 	 */
244 	lockdep_assert_held(&tl->mutex);
245 
246 	if (atomic_add_unless(&tl->active_count, 1, 0))
247 		return;
248 
249 	spin_lock(&timelines->lock);
250 	if (!atomic_fetch_inc(&tl->active_count)) {
251 		/*
252 		 * The HWSP is volatile, and may have been lost while inactive,
253 		 * e.g. across suspend/resume. Be paranoid, and ensure that
254 		 * the HWSP value matches our seqno so we don't proclaim
255 		 * the next request as already complete.
256 		 */
257 		intel_timeline_reset_seqno(tl);
258 		list_add_tail(&tl->link, &timelines->active_list);
259 	}
260 	spin_unlock(&timelines->lock);
261 }
262 
263 void intel_timeline_exit(struct intel_timeline *tl)
264 {
265 	struct intel_gt_timelines *timelines = &tl->gt->timelines;
266 
267 	/* See intel_timeline_enter() */
268 	lockdep_assert_held(&tl->mutex);
269 
270 	GEM_BUG_ON(!atomic_read(&tl->active_count));
271 	if (atomic_add_unless(&tl->active_count, -1, 1))
272 		return;
273 
274 	spin_lock(&timelines->lock);
275 	if (atomic_dec_and_test(&tl->active_count))
276 		list_del(&tl->link);
277 	spin_unlock(&timelines->lock);
278 
279 	/*
280 	 * Since this timeline is idle, all bariers upon which we were waiting
281 	 * must also be complete and so we can discard the last used barriers
282 	 * without loss of information.
283 	 */
284 	i915_syncmap_free(&tl->sync);
285 }
286 
287 static u32 timeline_advance(struct intel_timeline *tl)
288 {
289 	GEM_BUG_ON(!atomic_read(&tl->pin_count));
290 	GEM_BUG_ON(tl->seqno & tl->has_initial_breadcrumb);
291 
292 	return tl->seqno += 1 + tl->has_initial_breadcrumb;
293 }
294 
295 static noinline int
296 __intel_timeline_get_seqno(struct intel_timeline *tl,
297 			   u32 *seqno)
298 {
299 	u32 next_ofs = offset_in_page(tl->hwsp_offset + TIMELINE_SEQNO_BYTES);
300 
301 	/* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
302 	if (TIMELINE_SEQNO_BYTES <= BIT(5) && (next_ofs & BIT(5)))
303 		next_ofs = offset_in_page(next_ofs + BIT(5));
304 
305 	tl->hwsp_offset = i915_ggtt_offset(tl->hwsp_ggtt) + next_ofs;
306 	tl->hwsp_seqno = tl->hwsp_map + next_ofs;
307 	intel_timeline_reset_seqno(tl);
308 
309 	*seqno = timeline_advance(tl);
310 	GEM_BUG_ON(i915_seqno_passed(*tl->hwsp_seqno, *seqno));
311 	return 0;
312 }
313 
314 int intel_timeline_get_seqno(struct intel_timeline *tl,
315 			     struct i915_request *rq,
316 			     u32 *seqno)
317 {
318 	*seqno = timeline_advance(tl);
319 
320 	/* Replace the HWSP on wraparound for HW semaphores */
321 	if (unlikely(!*seqno && tl->has_initial_breadcrumb))
322 		return __intel_timeline_get_seqno(tl, seqno);
323 
324 	return 0;
325 }
326 
327 int intel_timeline_read_hwsp(struct i915_request *from,
328 			     struct i915_request *to,
329 			     u32 *hwsp)
330 {
331 	struct intel_timeline *tl;
332 	int err;
333 
334 	rcu_read_lock();
335 	tl = rcu_dereference(from->timeline);
336 	if (i915_request_signaled(from) ||
337 	    !i915_active_acquire_if_busy(&tl->active))
338 		tl = NULL;
339 
340 	if (tl) {
341 		/* hwsp_offset may wraparound, so use from->hwsp_seqno */
342 		*hwsp = i915_ggtt_offset(tl->hwsp_ggtt) +
343 			offset_in_page(from->hwsp_seqno);
344 	}
345 
346 	/* ensure we wait on the right request, if not, we completed */
347 	if (tl && __i915_request_is_complete(from)) {
348 		i915_active_release(&tl->active);
349 		tl = NULL;
350 	}
351 	rcu_read_unlock();
352 
353 	if (!tl)
354 		return 1;
355 
356 	/* Can't do semaphore waits on kernel context */
357 	if (!tl->has_initial_breadcrumb) {
358 		err = -EINVAL;
359 		goto out;
360 	}
361 
362 	err = i915_active_add_request(&tl->active, to);
363 
364 out:
365 	i915_active_release(&tl->active);
366 	return err;
367 }
368 
369 void intel_timeline_unpin(struct intel_timeline *tl)
370 {
371 	GEM_BUG_ON(!atomic_read(&tl->pin_count));
372 	if (!atomic_dec_and_test(&tl->pin_count))
373 		return;
374 
375 	i915_active_release(&tl->active);
376 	__i915_vma_unpin(tl->hwsp_ggtt);
377 }
378 
379 void __intel_timeline_free(struct kref *kref)
380 {
381 	struct intel_timeline *timeline =
382 		container_of(kref, typeof(*timeline), kref);
383 
384 	GEM_BUG_ON(atomic_read(&timeline->pin_count));
385 	GEM_BUG_ON(!list_empty(&timeline->requests));
386 	GEM_BUG_ON(timeline->retire);
387 
388 	call_rcu(&timeline->rcu, intel_timeline_fini);
389 }
390 
391 void intel_gt_fini_timelines(struct intel_gt *gt)
392 {
393 	struct intel_gt_timelines *timelines = &gt->timelines;
394 
395 	GEM_BUG_ON(!list_empty(&timelines->active_list));
396 }
397 
398 void intel_gt_show_timelines(struct intel_gt *gt,
399 			     struct drm_printer *m,
400 			     void (*show_request)(struct drm_printer *m,
401 						  const struct i915_request *rq,
402 						  const char *prefix,
403 						  int indent))
404 {
405 	struct intel_gt_timelines *timelines = &gt->timelines;
406 	struct intel_timeline *tl, *tn;
407 	LIST_HEAD(free);
408 
409 	spin_lock(&timelines->lock);
410 	list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {
411 		unsigned long count, ready, inflight;
412 		struct i915_request *rq, *rn;
413 		struct dma_fence *fence;
414 
415 		if (!mutex_trylock(&tl->mutex)) {
416 			drm_printf(m, "Timeline %llx: busy; skipping\n",
417 				   tl->fence_context);
418 			continue;
419 		}
420 
421 		intel_timeline_get(tl);
422 		GEM_BUG_ON(!atomic_read(&tl->active_count));
423 		atomic_inc(&tl->active_count); /* pin the list element */
424 		spin_unlock(&timelines->lock);
425 
426 		count = 0;
427 		ready = 0;
428 		inflight = 0;
429 		list_for_each_entry_safe(rq, rn, &tl->requests, link) {
430 			if (i915_request_completed(rq))
431 				continue;
432 
433 			count++;
434 			if (i915_request_is_ready(rq))
435 				ready++;
436 			if (i915_request_is_active(rq))
437 				inflight++;
438 		}
439 
440 		drm_printf(m, "Timeline %llx: { ", tl->fence_context);
441 		drm_printf(m, "count: %lu, ready: %lu, inflight: %lu",
442 			   count, ready, inflight);
443 		drm_printf(m, ", seqno: { current: %d, last: %d }",
444 			   *tl->hwsp_seqno, tl->seqno);
445 		fence = i915_active_fence_get(&tl->last_request);
446 		if (fence) {
447 			drm_printf(m, ", engine: %s",
448 				   to_request(fence)->engine->name);
449 			dma_fence_put(fence);
450 		}
451 		drm_printf(m, " }\n");
452 
453 		if (show_request) {
454 			list_for_each_entry_safe(rq, rn, &tl->requests, link)
455 				show_request(m, rq, "", 2);
456 		}
457 
458 		mutex_unlock(&tl->mutex);
459 		spin_lock(&timelines->lock);
460 
461 		/* Resume list iteration after reacquiring spinlock */
462 		list_safe_reset_next(tl, tn, link);
463 		if (atomic_dec_and_test(&tl->active_count))
464 			list_del(&tl->link);
465 
466 		/* Defer the final release to after the spinlock */
467 		if (refcount_dec_and_test(&tl->kref.refcount)) {
468 			GEM_BUG_ON(atomic_read(&tl->active_count));
469 			list_add(&tl->link, &free);
470 		}
471 	}
472 	spin_unlock(&timelines->lock);
473 
474 	list_for_each_entry_safe(tl, tn, &free, link)
475 		__intel_timeline_free(&tl->kref);
476 }
477 
478 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
479 #include "gt/selftests/mock_timeline.c"
480 #include "gt/selftest_timeline.c"
481 #endif
482