1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2019 Intel Corporation
5  */
6 
7 #include "gem/i915_gem_context.h"
8 #include "gem/i915_gem_pm.h"
9 
10 #include "i915_drv.h"
11 #include "i915_globals.h"
12 
13 #include "intel_context.h"
14 #include "intel_engine.h"
15 #include "intel_engine_pm.h"
16 #include "intel_ring.h"
17 
18 static struct i915_global_context {
19 	struct i915_global base;
20 	struct kmem_cache *slab_ce;
21 } global;
22 
23 static struct intel_context *intel_context_alloc(void)
24 {
25 	return kmem_cache_zalloc(global.slab_ce, GFP_KERNEL);
26 }
27 
28 void intel_context_free(struct intel_context *ce)
29 {
30 	kmem_cache_free(global.slab_ce, ce);
31 }
32 
33 struct intel_context *
34 intel_context_create(struct intel_engine_cs *engine)
35 {
36 	struct intel_context *ce;
37 
38 	ce = intel_context_alloc();
39 	if (!ce)
40 		return ERR_PTR(-ENOMEM);
41 
42 	intel_context_init(ce, engine);
43 	return ce;
44 }
45 
46 int intel_context_alloc_state(struct intel_context *ce)
47 {
48 	int err = 0;
49 
50 	if (mutex_lock_interruptible(&ce->pin_mutex))
51 		return -EINTR;
52 
53 	if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) {
54 		if (intel_context_is_banned(ce)) {
55 			err = -EIO;
56 			goto unlock;
57 		}
58 
59 		err = ce->ops->alloc(ce);
60 		if (unlikely(err))
61 			goto unlock;
62 
63 		set_bit(CONTEXT_ALLOC_BIT, &ce->flags);
64 	}
65 
66 unlock:
67 	mutex_unlock(&ce->pin_mutex);
68 	return err;
69 }
70 
71 static int intel_context_active_acquire(struct intel_context *ce)
72 {
73 	int err;
74 
75 	__i915_active_acquire(&ce->active);
76 
77 	if (intel_context_is_barrier(ce))
78 		return 0;
79 
80 	/* Preallocate tracking nodes */
81 	err = i915_active_acquire_preallocate_barrier(&ce->active,
82 						      ce->engine);
83 	if (err)
84 		i915_active_release(&ce->active);
85 
86 	return err;
87 }
88 
89 static void intel_context_active_release(struct intel_context *ce)
90 {
91 	/* Nodes preallocated in intel_context_active() */
92 	i915_active_acquire_barrier(&ce->active);
93 	i915_active_release(&ce->active);
94 }
95 
96 int __intel_context_do_pin(struct intel_context *ce)
97 {
98 	int err;
99 
100 	GEM_BUG_ON(intel_context_is_closed(ce));
101 
102 	if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) {
103 		err = intel_context_alloc_state(ce);
104 		if (err)
105 			return err;
106 	}
107 
108 	err = i915_active_acquire(&ce->active);
109 	if (err)
110 		return err;
111 
112 	if (mutex_lock_interruptible(&ce->pin_mutex)) {
113 		err = -EINTR;
114 		goto out_release;
115 	}
116 
117 	if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) {
118 		err = intel_context_active_acquire(ce);
119 		if (unlikely(err))
120 			goto out_unlock;
121 
122 		err = ce->ops->pin(ce);
123 		if (unlikely(err))
124 			goto err_active;
125 
126 		CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n",
127 			 i915_ggtt_offset(ce->ring->vma),
128 			 ce->ring->head, ce->ring->tail);
129 
130 		smp_mb__before_atomic(); /* flush pin before it is visible */
131 		atomic_inc(&ce->pin_count);
132 	}
133 
134 	GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */
135 	GEM_BUG_ON(i915_active_is_idle(&ce->active));
136 	goto out_unlock;
137 
138 err_active:
139 	intel_context_active_release(ce);
140 out_unlock:
141 	mutex_unlock(&ce->pin_mutex);
142 out_release:
143 	i915_active_release(&ce->active);
144 	return err;
145 }
146 
147 void intel_context_unpin(struct intel_context *ce)
148 {
149 	if (!atomic_dec_and_test(&ce->pin_count))
150 		return;
151 
152 	CE_TRACE(ce, "unpin\n");
153 	ce->ops->unpin(ce);
154 
155 	/*
156 	 * Once released, we may asynchronously drop the active reference.
157 	 * As that may be the only reference keeping the context alive,
158 	 * take an extra now so that it is not freed before we finish
159 	 * dereferencing it.
160 	 */
161 	intel_context_get(ce);
162 	intel_context_active_release(ce);
163 	intel_context_put(ce);
164 }
165 
166 static int __context_pin_state(struct i915_vma *vma)
167 {
168 	unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS;
169 	int err;
170 
171 	err = i915_ggtt_pin(vma, 0, bias | PIN_HIGH);
172 	if (err)
173 		return err;
174 
175 	err = i915_active_acquire(&vma->active);
176 	if (err)
177 		goto err_unpin;
178 
179 	/*
180 	 * And mark it as a globally pinned object to let the shrinker know
181 	 * it cannot reclaim the object until we release it.
182 	 */
183 	i915_vma_make_unshrinkable(vma);
184 	vma->obj->mm.dirty = true;
185 
186 	return 0;
187 
188 err_unpin:
189 	i915_vma_unpin(vma);
190 	return err;
191 }
192 
193 static void __context_unpin_state(struct i915_vma *vma)
194 {
195 	i915_vma_make_shrinkable(vma);
196 	i915_active_release(&vma->active);
197 	__i915_vma_unpin(vma);
198 }
199 
200 static int __ring_active(struct intel_ring *ring)
201 {
202 	int err;
203 
204 	err = i915_active_acquire(&ring->vma->active);
205 	if (err)
206 		return err;
207 
208 	err = intel_ring_pin(ring);
209 	if (err)
210 		goto err_active;
211 
212 	return 0;
213 
214 err_active:
215 	i915_active_release(&ring->vma->active);
216 	return err;
217 }
218 
219 static void __ring_retire(struct intel_ring *ring)
220 {
221 	intel_ring_unpin(ring);
222 	i915_active_release(&ring->vma->active);
223 }
224 
225 __i915_active_call
226 static void __intel_context_retire(struct i915_active *active)
227 {
228 	struct intel_context *ce = container_of(active, typeof(*ce), active);
229 
230 	CE_TRACE(ce, "retire runtime: { total:%lluns, avg:%lluns }\n",
231 		 intel_context_get_total_runtime_ns(ce),
232 		 intel_context_get_avg_runtime_ns(ce));
233 
234 	set_bit(CONTEXT_VALID_BIT, &ce->flags);
235 	if (ce->state)
236 		__context_unpin_state(ce->state);
237 
238 	intel_timeline_unpin(ce->timeline);
239 	__ring_retire(ce->ring);
240 
241 	intel_context_put(ce);
242 }
243 
244 static int __intel_context_active(struct i915_active *active)
245 {
246 	struct intel_context *ce = container_of(active, typeof(*ce), active);
247 	int err;
248 
249 	CE_TRACE(ce, "active\n");
250 
251 	intel_context_get(ce);
252 
253 	err = __ring_active(ce->ring);
254 	if (err)
255 		goto err_put;
256 
257 	err = intel_timeline_pin(ce->timeline);
258 	if (err)
259 		goto err_ring;
260 
261 	if (!ce->state)
262 		return 0;
263 
264 	err = __context_pin_state(ce->state);
265 	if (err)
266 		goto err_timeline;
267 
268 	return 0;
269 
270 err_timeline:
271 	intel_timeline_unpin(ce->timeline);
272 err_ring:
273 	__ring_retire(ce->ring);
274 err_put:
275 	intel_context_put(ce);
276 	return err;
277 }
278 
279 void
280 intel_context_init(struct intel_context *ce,
281 		   struct intel_engine_cs *engine)
282 {
283 	GEM_BUG_ON(!engine->cops);
284 	GEM_BUG_ON(!engine->gt->vm);
285 
286 	kref_init(&ce->ref);
287 
288 	ce->engine = engine;
289 	ce->ops = engine->cops;
290 	ce->sseu = engine->sseu;
291 	ce->ring = __intel_context_ring_size(SZ_4K);
292 
293 	ewma_runtime_init(&ce->runtime.avg);
294 
295 	ce->vm = i915_vm_get(engine->gt->vm);
296 
297 	INIT_LIST_HEAD(&ce->signal_link);
298 	INIT_LIST_HEAD(&ce->signals);
299 
300 	mutex_init(&ce->pin_mutex);
301 
302 	i915_active_init(&ce->active,
303 			 __intel_context_active, __intel_context_retire);
304 }
305 
306 void intel_context_fini(struct intel_context *ce)
307 {
308 	if (ce->timeline)
309 		intel_timeline_put(ce->timeline);
310 	i915_vm_put(ce->vm);
311 
312 	mutex_destroy(&ce->pin_mutex);
313 	i915_active_fini(&ce->active);
314 }
315 
316 static void i915_global_context_shrink(void)
317 {
318 	kmem_cache_shrink(global.slab_ce);
319 }
320 
321 static void i915_global_context_exit(void)
322 {
323 	kmem_cache_destroy(global.slab_ce);
324 }
325 
326 static struct i915_global_context global = { {
327 	.shrink = i915_global_context_shrink,
328 	.exit = i915_global_context_exit,
329 } };
330 
331 int __init i915_global_context_init(void)
332 {
333 	global.slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN);
334 	if (!global.slab_ce)
335 		return -ENOMEM;
336 
337 	i915_global_register(&global.base);
338 	return 0;
339 }
340 
341 void intel_context_enter_engine(struct intel_context *ce)
342 {
343 	intel_engine_pm_get(ce->engine);
344 	intel_timeline_enter(ce->timeline);
345 }
346 
347 void intel_context_exit_engine(struct intel_context *ce)
348 {
349 	intel_timeline_exit(ce->timeline);
350 	intel_engine_pm_put(ce->engine);
351 }
352 
353 int intel_context_prepare_remote_request(struct intel_context *ce,
354 					 struct i915_request *rq)
355 {
356 	struct intel_timeline *tl = ce->timeline;
357 	int err;
358 
359 	/* Only suitable for use in remotely modifying this context */
360 	GEM_BUG_ON(rq->context == ce);
361 
362 	if (rcu_access_pointer(rq->timeline) != tl) { /* timeline sharing! */
363 		/* Queue this switch after current activity by this context. */
364 		err = i915_active_fence_set(&tl->last_request, rq);
365 		if (err)
366 			return err;
367 	}
368 
369 	/*
370 	 * Guarantee context image and the timeline remains pinned until the
371 	 * modifying request is retired by setting the ce activity tracker.
372 	 *
373 	 * But we only need to take one pin on the account of it. Or in other
374 	 * words transfer the pinned ce object to tracked active request.
375 	 */
376 	GEM_BUG_ON(i915_active_is_idle(&ce->active));
377 	return i915_active_add_request(&ce->active, rq);
378 }
379 
380 struct i915_request *intel_context_create_request(struct intel_context *ce)
381 {
382 	struct i915_request *rq;
383 	int err;
384 
385 	err = intel_context_pin(ce);
386 	if (unlikely(err))
387 		return ERR_PTR(err);
388 
389 	rq = i915_request_create(ce);
390 	intel_context_unpin(ce);
391 
392 	return rq;
393 }
394 
395 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
396 #include "selftest_context.c"
397 #endif
398