1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2019 Intel Corporation
5  */
6 
7 #include "gem/i915_gem_context.h"
8 #include "gem/i915_gem_pm.h"
9 
10 #include "i915_drv.h"
11 #include "i915_globals.h"
12 
13 #include "intel_context.h"
14 #include "intel_engine.h"
15 #include "intel_engine_pm.h"
16 #include "intel_ring.h"
17 
18 static struct i915_global_context {
19 	struct i915_global base;
20 	struct kmem_cache *slab_ce;
21 } global;
22 
23 static struct intel_context *intel_context_alloc(void)
24 {
25 	return kmem_cache_zalloc(global.slab_ce, GFP_KERNEL);
26 }
27 
28 void intel_context_free(struct intel_context *ce)
29 {
30 	kmem_cache_free(global.slab_ce, ce);
31 }
32 
33 struct intel_context *
34 intel_context_create(struct intel_engine_cs *engine)
35 {
36 	struct intel_context *ce;
37 
38 	ce = intel_context_alloc();
39 	if (!ce)
40 		return ERR_PTR(-ENOMEM);
41 
42 	intel_context_init(ce, engine);
43 	return ce;
44 }
45 
46 int intel_context_alloc_state(struct intel_context *ce)
47 {
48 	int err = 0;
49 
50 	if (mutex_lock_interruptible(&ce->pin_mutex))
51 		return -EINTR;
52 
53 	if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) {
54 		if (intel_context_is_banned(ce)) {
55 			err = -EIO;
56 			goto unlock;
57 		}
58 
59 		err = ce->ops->alloc(ce);
60 		if (unlikely(err))
61 			goto unlock;
62 
63 		set_bit(CONTEXT_ALLOC_BIT, &ce->flags);
64 	}
65 
66 unlock:
67 	mutex_unlock(&ce->pin_mutex);
68 	return err;
69 }
70 
71 static int intel_context_active_acquire(struct intel_context *ce)
72 {
73 	int err;
74 
75 	__i915_active_acquire(&ce->active);
76 
77 	if (intel_context_is_barrier(ce))
78 		return 0;
79 
80 	/* Preallocate tracking nodes */
81 	err = i915_active_acquire_preallocate_barrier(&ce->active,
82 						      ce->engine);
83 	if (err)
84 		i915_active_release(&ce->active);
85 
86 	return err;
87 }
88 
89 static void intel_context_active_release(struct intel_context *ce)
90 {
91 	/* Nodes preallocated in intel_context_active() */
92 	i915_active_acquire_barrier(&ce->active);
93 	i915_active_release(&ce->active);
94 }
95 
96 static int __context_pin_state(struct i915_vma *vma, struct i915_gem_ww_ctx *ww)
97 {
98 	unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS;
99 	int err;
100 
101 	err = i915_ggtt_pin(vma, ww, 0, bias | PIN_HIGH);
102 	if (err)
103 		return err;
104 
105 	err = i915_active_acquire(&vma->active);
106 	if (err)
107 		goto err_unpin;
108 
109 	/*
110 	 * And mark it as a globally pinned object to let the shrinker know
111 	 * it cannot reclaim the object until we release it.
112 	 */
113 	i915_vma_make_unshrinkable(vma);
114 	vma->obj->mm.dirty = true;
115 
116 	return 0;
117 
118 err_unpin:
119 	i915_vma_unpin(vma);
120 	return err;
121 }
122 
123 static void __context_unpin_state(struct i915_vma *vma)
124 {
125 	i915_vma_make_shrinkable(vma);
126 	i915_active_release(&vma->active);
127 	__i915_vma_unpin(vma);
128 }
129 
130 static int __ring_active(struct intel_ring *ring,
131 			 struct i915_gem_ww_ctx *ww)
132 {
133 	int err;
134 
135 	err = intel_ring_pin(ring, ww);
136 	if (err)
137 		return err;
138 
139 	err = i915_active_acquire(&ring->vma->active);
140 	if (err)
141 		goto err_pin;
142 
143 	return 0;
144 
145 err_pin:
146 	intel_ring_unpin(ring);
147 	return err;
148 }
149 
150 static void __ring_retire(struct intel_ring *ring)
151 {
152 	i915_active_release(&ring->vma->active);
153 	intel_ring_unpin(ring);
154 }
155 
156 static int intel_context_pre_pin(struct intel_context *ce,
157 				 struct i915_gem_ww_ctx *ww)
158 {
159 	int err;
160 
161 	CE_TRACE(ce, "active\n");
162 
163 	err = __ring_active(ce->ring, ww);
164 	if (err)
165 		return err;
166 
167 	err = intel_timeline_pin(ce->timeline, ww);
168 	if (err)
169 		goto err_ring;
170 
171 	if (!ce->state)
172 		return 0;
173 
174 	err = __context_pin_state(ce->state, ww);
175 	if (err)
176 		goto err_timeline;
177 
178 
179 	return 0;
180 
181 err_timeline:
182 	intel_timeline_unpin(ce->timeline);
183 err_ring:
184 	__ring_retire(ce->ring);
185 	return err;
186 }
187 
188 static void intel_context_post_unpin(struct intel_context *ce)
189 {
190 	if (ce->state)
191 		__context_unpin_state(ce->state);
192 
193 	intel_timeline_unpin(ce->timeline);
194 	__ring_retire(ce->ring);
195 }
196 
197 int __intel_context_do_pin_ww(struct intel_context *ce,
198 			      struct i915_gem_ww_ctx *ww)
199 {
200 	bool handoff = false;
201 	void *vaddr;
202 	int err = 0;
203 
204 	if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) {
205 		err = intel_context_alloc_state(ce);
206 		if (err)
207 			return err;
208 	}
209 
210 	/*
211 	 * We always pin the context/ring/timeline here, to ensure a pin
212 	 * refcount for __intel_context_active(), which prevent a lock
213 	 * inversion of ce->pin_mutex vs dma_resv_lock().
214 	 */
215 
216 	err = i915_gem_object_lock(ce->timeline->hwsp_ggtt->obj, ww);
217 	if (!err && ce->ring->vma->obj)
218 		err = i915_gem_object_lock(ce->ring->vma->obj, ww);
219 	if (!err && ce->state)
220 		err = i915_gem_object_lock(ce->state->obj, ww);
221 	if (!err)
222 		err = intel_context_pre_pin(ce, ww);
223 	if (err)
224 		return err;
225 
226 	err = i915_active_acquire(&ce->active);
227 	if (err)
228 		goto err_ctx_unpin;
229 
230 	err = ce->ops->pre_pin(ce, ww, &vaddr);
231 	if (err)
232 		goto err_release;
233 
234 	err = mutex_lock_interruptible(&ce->pin_mutex);
235 	if (err)
236 		goto err_post_unpin;
237 
238 	if (unlikely(intel_context_is_closed(ce))) {
239 		err = -ENOENT;
240 		goto err_unlock;
241 	}
242 
243 	if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) {
244 		err = intel_context_active_acquire(ce);
245 		if (unlikely(err))
246 			goto err_unlock;
247 
248 		err = ce->ops->pin(ce, vaddr);
249 		if (err) {
250 			intel_context_active_release(ce);
251 			goto err_unlock;
252 		}
253 
254 		CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n",
255 			 i915_ggtt_offset(ce->ring->vma),
256 			 ce->ring->head, ce->ring->tail);
257 
258 		handoff = true;
259 		smp_mb__before_atomic(); /* flush pin before it is visible */
260 		atomic_inc(&ce->pin_count);
261 	}
262 
263 	GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */
264 
265 err_unlock:
266 	mutex_unlock(&ce->pin_mutex);
267 err_post_unpin:
268 	if (!handoff)
269 		ce->ops->post_unpin(ce);
270 err_release:
271 	i915_active_release(&ce->active);
272 err_ctx_unpin:
273 	intel_context_post_unpin(ce);
274 
275 	/*
276 	 * Unlock the hwsp_ggtt object since it's shared.
277 	 * In principle we can unlock all the global state locked above
278 	 * since it's pinned and doesn't need fencing, and will
279 	 * thus remain resident until it is explicitly unpinned.
280 	 */
281 	i915_gem_ww_unlock_single(ce->timeline->hwsp_ggtt->obj);
282 
283 	return err;
284 }
285 
286 int __intel_context_do_pin(struct intel_context *ce)
287 {
288 	struct i915_gem_ww_ctx ww;
289 	int err;
290 
291 	i915_gem_ww_ctx_init(&ww, true);
292 retry:
293 	err = __intel_context_do_pin_ww(ce, &ww);
294 	if (err == -EDEADLK) {
295 		err = i915_gem_ww_ctx_backoff(&ww);
296 		if (!err)
297 			goto retry;
298 	}
299 	i915_gem_ww_ctx_fini(&ww);
300 	return err;
301 }
302 
303 void intel_context_unpin(struct intel_context *ce)
304 {
305 	if (!atomic_dec_and_test(&ce->pin_count))
306 		return;
307 
308 	CE_TRACE(ce, "unpin\n");
309 	ce->ops->unpin(ce);
310 	ce->ops->post_unpin(ce);
311 
312 	/*
313 	 * Once released, we may asynchronously drop the active reference.
314 	 * As that may be the only reference keeping the context alive,
315 	 * take an extra now so that it is not freed before we finish
316 	 * dereferencing it.
317 	 */
318 	intel_context_get(ce);
319 	intel_context_active_release(ce);
320 	intel_context_put(ce);
321 }
322 
323 __i915_active_call
324 static void __intel_context_retire(struct i915_active *active)
325 {
326 	struct intel_context *ce = container_of(active, typeof(*ce), active);
327 
328 	CE_TRACE(ce, "retire runtime: { total:%lluns, avg:%lluns }\n",
329 		 intel_context_get_total_runtime_ns(ce),
330 		 intel_context_get_avg_runtime_ns(ce));
331 
332 	set_bit(CONTEXT_VALID_BIT, &ce->flags);
333 	intel_context_post_unpin(ce);
334 	intel_context_put(ce);
335 }
336 
337 static int __intel_context_active(struct i915_active *active)
338 {
339 	struct intel_context *ce = container_of(active, typeof(*ce), active);
340 
341 	intel_context_get(ce);
342 
343 	/* everything should already be activated by intel_context_pre_pin() */
344 	GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->ring->vma->active));
345 	__intel_ring_pin(ce->ring);
346 
347 	__intel_timeline_pin(ce->timeline);
348 
349 	if (ce->state) {
350 		GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->state->active));
351 		__i915_vma_pin(ce->state);
352 		i915_vma_make_unshrinkable(ce->state);
353 	}
354 
355 	return 0;
356 }
357 
358 void
359 intel_context_init(struct intel_context *ce,
360 		   struct intel_engine_cs *engine)
361 {
362 	GEM_BUG_ON(!engine->cops);
363 	GEM_BUG_ON(!engine->gt->vm);
364 
365 	kref_init(&ce->ref);
366 
367 	ce->engine = engine;
368 	ce->ops = engine->cops;
369 	ce->sseu = engine->sseu;
370 	ce->ring = __intel_context_ring_size(SZ_4K);
371 
372 	ewma_runtime_init(&ce->runtime.avg);
373 
374 	ce->vm = i915_vm_get(engine->gt->vm);
375 
376 	INIT_LIST_HEAD(&ce->signal_link);
377 	INIT_LIST_HEAD(&ce->signals);
378 
379 	mutex_init(&ce->pin_mutex);
380 
381 	i915_active_init(&ce->active,
382 			 __intel_context_active, __intel_context_retire);
383 }
384 
385 void intel_context_fini(struct intel_context *ce)
386 {
387 	if (ce->timeline)
388 		intel_timeline_put(ce->timeline);
389 	i915_vm_put(ce->vm);
390 
391 	mutex_destroy(&ce->pin_mutex);
392 	i915_active_fini(&ce->active);
393 }
394 
395 static void i915_global_context_shrink(void)
396 {
397 	kmem_cache_shrink(global.slab_ce);
398 }
399 
400 static void i915_global_context_exit(void)
401 {
402 	kmem_cache_destroy(global.slab_ce);
403 }
404 
405 static struct i915_global_context global = { {
406 	.shrink = i915_global_context_shrink,
407 	.exit = i915_global_context_exit,
408 } };
409 
410 int __init i915_global_context_init(void)
411 {
412 	global.slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN);
413 	if (!global.slab_ce)
414 		return -ENOMEM;
415 
416 	i915_global_register(&global.base);
417 	return 0;
418 }
419 
420 void intel_context_enter_engine(struct intel_context *ce)
421 {
422 	intel_engine_pm_get(ce->engine);
423 	intel_timeline_enter(ce->timeline);
424 }
425 
426 void intel_context_exit_engine(struct intel_context *ce)
427 {
428 	intel_timeline_exit(ce->timeline);
429 	intel_engine_pm_put(ce->engine);
430 }
431 
432 int intel_context_prepare_remote_request(struct intel_context *ce,
433 					 struct i915_request *rq)
434 {
435 	struct intel_timeline *tl = ce->timeline;
436 	int err;
437 
438 	/* Only suitable for use in remotely modifying this context */
439 	GEM_BUG_ON(rq->context == ce);
440 
441 	if (rcu_access_pointer(rq->timeline) != tl) { /* timeline sharing! */
442 		/* Queue this switch after current activity by this context. */
443 		err = i915_active_fence_set(&tl->last_request, rq);
444 		if (err)
445 			return err;
446 	}
447 
448 	/*
449 	 * Guarantee context image and the timeline remains pinned until the
450 	 * modifying request is retired by setting the ce activity tracker.
451 	 *
452 	 * But we only need to take one pin on the account of it. Or in other
453 	 * words transfer the pinned ce object to tracked active request.
454 	 */
455 	GEM_BUG_ON(i915_active_is_idle(&ce->active));
456 	return i915_active_add_request(&ce->active, rq);
457 }
458 
459 struct i915_request *intel_context_create_request(struct intel_context *ce)
460 {
461 	struct i915_gem_ww_ctx ww;
462 	struct i915_request *rq;
463 	int err;
464 
465 	i915_gem_ww_ctx_init(&ww, true);
466 retry:
467 	err = intel_context_pin_ww(ce, &ww);
468 	if (!err) {
469 		rq = i915_request_create(ce);
470 		intel_context_unpin(ce);
471 	} else if (err == -EDEADLK) {
472 		err = i915_gem_ww_ctx_backoff(&ww);
473 		if (!err)
474 			goto retry;
475 		rq = ERR_PTR(err);
476 	} else {
477 		rq = ERR_PTR(err);
478 	}
479 
480 	i915_gem_ww_ctx_fini(&ww);
481 
482 	if (IS_ERR(rq))
483 		return rq;
484 
485 	/*
486 	 * timeline->mutex should be the inner lock, but is used as outer lock.
487 	 * Hack around this to shut up lockdep in selftests..
488 	 */
489 	lockdep_unpin_lock(&ce->timeline->mutex, rq->cookie);
490 	mutex_release(&ce->timeline->mutex.dep_map, _RET_IP_);
491 	mutex_acquire(&ce->timeline->mutex.dep_map, SINGLE_DEPTH_NESTING, 0, _RET_IP_);
492 	rq->cookie = lockdep_pin_lock(&ce->timeline->mutex);
493 
494 	return rq;
495 }
496 
497 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
498 #include "selftest_context.c"
499 #endif
500