1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2019 Intel Corporation
5  */
6 
7 #include "gem/i915_gem_context.h"
8 #include "gem/i915_gem_pm.h"
9 
10 #include "i915_drv.h"
11 #include "i915_globals.h"
12 
13 #include "intel_context.h"
14 #include "intel_engine.h"
15 #include "intel_engine_pm.h"
16 #include "intel_ring.h"
17 
18 static struct i915_global_context {
19 	struct i915_global base;
20 	struct kmem_cache *slab_ce;
21 } global;
22 
23 static struct intel_context *intel_context_alloc(void)
24 {
25 	return kmem_cache_zalloc(global.slab_ce, GFP_KERNEL);
26 }
27 
28 void intel_context_free(struct intel_context *ce)
29 {
30 	kmem_cache_free(global.slab_ce, ce);
31 }
32 
33 struct intel_context *
34 intel_context_create(struct intel_engine_cs *engine)
35 {
36 	struct intel_context *ce;
37 
38 	ce = intel_context_alloc();
39 	if (!ce)
40 		return ERR_PTR(-ENOMEM);
41 
42 	intel_context_init(ce, engine);
43 	return ce;
44 }
45 
46 int intel_context_alloc_state(struct intel_context *ce)
47 {
48 	int err = 0;
49 
50 	if (mutex_lock_interruptible(&ce->pin_mutex))
51 		return -EINTR;
52 
53 	if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) {
54 		if (intel_context_is_banned(ce)) {
55 			err = -EIO;
56 			goto unlock;
57 		}
58 
59 		err = ce->ops->alloc(ce);
60 		if (unlikely(err))
61 			goto unlock;
62 
63 		set_bit(CONTEXT_ALLOC_BIT, &ce->flags);
64 	}
65 
66 unlock:
67 	mutex_unlock(&ce->pin_mutex);
68 	return err;
69 }
70 
71 static int intel_context_active_acquire(struct intel_context *ce)
72 {
73 	int err;
74 
75 	__i915_active_acquire(&ce->active);
76 
77 	if (intel_context_is_barrier(ce))
78 		return 0;
79 
80 	/* Preallocate tracking nodes */
81 	err = i915_active_acquire_preallocate_barrier(&ce->active,
82 						      ce->engine);
83 	if (err)
84 		i915_active_release(&ce->active);
85 
86 	return err;
87 }
88 
89 static void intel_context_active_release(struct intel_context *ce)
90 {
91 	/* Nodes preallocated in intel_context_active() */
92 	i915_active_acquire_barrier(&ce->active);
93 	i915_active_release(&ce->active);
94 }
95 
96 int __intel_context_do_pin(struct intel_context *ce)
97 {
98 	int err;
99 
100 	GEM_BUG_ON(intel_context_is_closed(ce));
101 
102 	if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) {
103 		err = intel_context_alloc_state(ce);
104 		if (err)
105 			return err;
106 	}
107 
108 	err = i915_active_acquire(&ce->active);
109 	if (err)
110 		return err;
111 
112 	if (mutex_lock_interruptible(&ce->pin_mutex)) {
113 		err = -EINTR;
114 		goto out_release;
115 	}
116 
117 	if (unlikely(intel_context_is_closed(ce))) {
118 		err = -ENOENT;
119 		goto out_unlock;
120 	}
121 
122 	if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) {
123 		err = intel_context_active_acquire(ce);
124 		if (unlikely(err))
125 			goto out_unlock;
126 
127 		err = ce->ops->pin(ce);
128 		if (unlikely(err))
129 			goto err_active;
130 
131 		CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n",
132 			 i915_ggtt_offset(ce->ring->vma),
133 			 ce->ring->head, ce->ring->tail);
134 
135 		smp_mb__before_atomic(); /* flush pin before it is visible */
136 		atomic_inc(&ce->pin_count);
137 	}
138 
139 	GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */
140 	GEM_BUG_ON(i915_active_is_idle(&ce->active));
141 	goto out_unlock;
142 
143 err_active:
144 	intel_context_active_release(ce);
145 out_unlock:
146 	mutex_unlock(&ce->pin_mutex);
147 out_release:
148 	i915_active_release(&ce->active);
149 	return err;
150 }
151 
152 void intel_context_unpin(struct intel_context *ce)
153 {
154 	if (!atomic_dec_and_test(&ce->pin_count))
155 		return;
156 
157 	CE_TRACE(ce, "unpin\n");
158 	ce->ops->unpin(ce);
159 
160 	/*
161 	 * Once released, we may asynchronously drop the active reference.
162 	 * As that may be the only reference keeping the context alive,
163 	 * take an extra now so that it is not freed before we finish
164 	 * dereferencing it.
165 	 */
166 	intel_context_get(ce);
167 	intel_context_active_release(ce);
168 	intel_context_put(ce);
169 }
170 
171 static int __context_pin_state(struct i915_vma *vma)
172 {
173 	unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS;
174 	int err;
175 
176 	err = i915_ggtt_pin(vma, 0, bias | PIN_HIGH);
177 	if (err)
178 		return err;
179 
180 	err = i915_active_acquire(&vma->active);
181 	if (err)
182 		goto err_unpin;
183 
184 	/*
185 	 * And mark it as a globally pinned object to let the shrinker know
186 	 * it cannot reclaim the object until we release it.
187 	 */
188 	i915_vma_make_unshrinkable(vma);
189 	vma->obj->mm.dirty = true;
190 
191 	return 0;
192 
193 err_unpin:
194 	i915_vma_unpin(vma);
195 	return err;
196 }
197 
198 static void __context_unpin_state(struct i915_vma *vma)
199 {
200 	i915_vma_make_shrinkable(vma);
201 	i915_active_release(&vma->active);
202 	__i915_vma_unpin(vma);
203 }
204 
205 static int __ring_active(struct intel_ring *ring)
206 {
207 	int err;
208 
209 	err = i915_active_acquire(&ring->vma->active);
210 	if (err)
211 		return err;
212 
213 	err = intel_ring_pin(ring);
214 	if (err)
215 		goto err_active;
216 
217 	return 0;
218 
219 err_active:
220 	i915_active_release(&ring->vma->active);
221 	return err;
222 }
223 
224 static void __ring_retire(struct intel_ring *ring)
225 {
226 	intel_ring_unpin(ring);
227 	i915_active_release(&ring->vma->active);
228 }
229 
230 __i915_active_call
231 static void __intel_context_retire(struct i915_active *active)
232 {
233 	struct intel_context *ce = container_of(active, typeof(*ce), active);
234 
235 	CE_TRACE(ce, "retire runtime: { total:%lluns, avg:%lluns }\n",
236 		 intel_context_get_total_runtime_ns(ce),
237 		 intel_context_get_avg_runtime_ns(ce));
238 
239 	set_bit(CONTEXT_VALID_BIT, &ce->flags);
240 	if (ce->state)
241 		__context_unpin_state(ce->state);
242 
243 	intel_timeline_unpin(ce->timeline);
244 	__ring_retire(ce->ring);
245 
246 	intel_context_put(ce);
247 }
248 
249 static int __intel_context_active(struct i915_active *active)
250 {
251 	struct intel_context *ce = container_of(active, typeof(*ce), active);
252 	int err;
253 
254 	CE_TRACE(ce, "active\n");
255 
256 	intel_context_get(ce);
257 
258 	err = __ring_active(ce->ring);
259 	if (err)
260 		goto err_put;
261 
262 	err = intel_timeline_pin(ce->timeline);
263 	if (err)
264 		goto err_ring;
265 
266 	if (!ce->state)
267 		return 0;
268 
269 	err = __context_pin_state(ce->state);
270 	if (err)
271 		goto err_timeline;
272 
273 	return 0;
274 
275 err_timeline:
276 	intel_timeline_unpin(ce->timeline);
277 err_ring:
278 	__ring_retire(ce->ring);
279 err_put:
280 	intel_context_put(ce);
281 	return err;
282 }
283 
284 void
285 intel_context_init(struct intel_context *ce,
286 		   struct intel_engine_cs *engine)
287 {
288 	GEM_BUG_ON(!engine->cops);
289 	GEM_BUG_ON(!engine->gt->vm);
290 
291 	kref_init(&ce->ref);
292 
293 	ce->engine = engine;
294 	ce->ops = engine->cops;
295 	ce->sseu = engine->sseu;
296 	ce->ring = __intel_context_ring_size(SZ_4K);
297 
298 	ewma_runtime_init(&ce->runtime.avg);
299 
300 	ce->vm = i915_vm_get(engine->gt->vm);
301 
302 	INIT_LIST_HEAD(&ce->signal_link);
303 	INIT_LIST_HEAD(&ce->signals);
304 
305 	mutex_init(&ce->pin_mutex);
306 
307 	i915_active_init(&ce->active,
308 			 __intel_context_active, __intel_context_retire);
309 }
310 
311 void intel_context_fini(struct intel_context *ce)
312 {
313 	if (ce->timeline)
314 		intel_timeline_put(ce->timeline);
315 	i915_vm_put(ce->vm);
316 
317 	mutex_destroy(&ce->pin_mutex);
318 	i915_active_fini(&ce->active);
319 }
320 
321 static void i915_global_context_shrink(void)
322 {
323 	kmem_cache_shrink(global.slab_ce);
324 }
325 
326 static void i915_global_context_exit(void)
327 {
328 	kmem_cache_destroy(global.slab_ce);
329 }
330 
331 static struct i915_global_context global = { {
332 	.shrink = i915_global_context_shrink,
333 	.exit = i915_global_context_exit,
334 } };
335 
336 int __init i915_global_context_init(void)
337 {
338 	global.slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN);
339 	if (!global.slab_ce)
340 		return -ENOMEM;
341 
342 	i915_global_register(&global.base);
343 	return 0;
344 }
345 
346 void intel_context_enter_engine(struct intel_context *ce)
347 {
348 	intel_engine_pm_get(ce->engine);
349 	intel_timeline_enter(ce->timeline);
350 }
351 
352 void intel_context_exit_engine(struct intel_context *ce)
353 {
354 	intel_timeline_exit(ce->timeline);
355 	intel_engine_pm_put(ce->engine);
356 }
357 
358 int intel_context_prepare_remote_request(struct intel_context *ce,
359 					 struct i915_request *rq)
360 {
361 	struct intel_timeline *tl = ce->timeline;
362 	int err;
363 
364 	/* Only suitable for use in remotely modifying this context */
365 	GEM_BUG_ON(rq->context == ce);
366 
367 	if (rcu_access_pointer(rq->timeline) != tl) { /* timeline sharing! */
368 		/* Queue this switch after current activity by this context. */
369 		err = i915_active_fence_set(&tl->last_request, rq);
370 		if (err)
371 			return err;
372 	}
373 
374 	/*
375 	 * Guarantee context image and the timeline remains pinned until the
376 	 * modifying request is retired by setting the ce activity tracker.
377 	 *
378 	 * But we only need to take one pin on the account of it. Or in other
379 	 * words transfer the pinned ce object to tracked active request.
380 	 */
381 	GEM_BUG_ON(i915_active_is_idle(&ce->active));
382 	return i915_active_add_request(&ce->active, rq);
383 }
384 
385 struct i915_request *intel_context_create_request(struct intel_context *ce)
386 {
387 	struct i915_request *rq;
388 	int err;
389 
390 	err = intel_context_pin(ce);
391 	if (unlikely(err))
392 		return ERR_PTR(err);
393 
394 	rq = i915_request_create(ce);
395 	intel_context_unpin(ce);
396 
397 	return rq;
398 }
399 
400 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
401 #include "selftest_context.c"
402 #endif
403