1 /*
2  * Copyright 2015 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: monk liu <monk.liu@amd.com>
23  */
24 
25 #include <drm/drm_auth.h>
26 #include "amdgpu.h"
27 #include "amdgpu_sched.h"
28 #include "amdgpu_ras.h"
29 #include <linux/nospec.h>
30 
31 #define to_amdgpu_ctx_entity(e)	\
32 	container_of((e), struct amdgpu_ctx_entity, entity)
33 
34 const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM] = {
35 	[AMDGPU_HW_IP_GFX]	=	1,
36 	[AMDGPU_HW_IP_COMPUTE]	=	4,
37 	[AMDGPU_HW_IP_DMA]	=	2,
38 	[AMDGPU_HW_IP_UVD]	=	1,
39 	[AMDGPU_HW_IP_VCE]	=	1,
40 	[AMDGPU_HW_IP_UVD_ENC]	=	1,
41 	[AMDGPU_HW_IP_VCN_DEC]	=	1,
42 	[AMDGPU_HW_IP_VCN_ENC]	=	1,
43 	[AMDGPU_HW_IP_VCN_JPEG]	=	1,
44 };
45 
46 bool amdgpu_ctx_priority_is_valid(int32_t ctx_prio)
47 {
48 	switch (ctx_prio) {
49 	case AMDGPU_CTX_PRIORITY_UNSET:
50 	case AMDGPU_CTX_PRIORITY_VERY_LOW:
51 	case AMDGPU_CTX_PRIORITY_LOW:
52 	case AMDGPU_CTX_PRIORITY_NORMAL:
53 	case AMDGPU_CTX_PRIORITY_HIGH:
54 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
55 		return true;
56 	default:
57 		return false;
58 	}
59 }
60 
61 static enum drm_sched_priority
62 amdgpu_ctx_to_drm_sched_prio(int32_t ctx_prio)
63 {
64 	switch (ctx_prio) {
65 	case AMDGPU_CTX_PRIORITY_UNSET:
66 		return DRM_SCHED_PRIORITY_UNSET;
67 
68 	case AMDGPU_CTX_PRIORITY_VERY_LOW:
69 		return DRM_SCHED_PRIORITY_MIN;
70 
71 	case AMDGPU_CTX_PRIORITY_LOW:
72 		return DRM_SCHED_PRIORITY_MIN;
73 
74 	case AMDGPU_CTX_PRIORITY_NORMAL:
75 		return DRM_SCHED_PRIORITY_NORMAL;
76 
77 	case AMDGPU_CTX_PRIORITY_HIGH:
78 		return DRM_SCHED_PRIORITY_HIGH;
79 
80 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
81 		return DRM_SCHED_PRIORITY_HIGH;
82 
83 	/* This should not happen as we sanitized userspace provided priority
84 	 * already, WARN if this happens.
85 	 */
86 	default:
87 		WARN(1, "Invalid context priority %d\n", ctx_prio);
88 		return DRM_SCHED_PRIORITY_NORMAL;
89 	}
90 
91 }
92 
93 static int amdgpu_ctx_priority_permit(struct drm_file *filp,
94 				      int32_t priority)
95 {
96 	if (!amdgpu_ctx_priority_is_valid(priority))
97 		return -EINVAL;
98 
99 	/* NORMAL and below are accessible by everyone */
100 	if (priority <= AMDGPU_CTX_PRIORITY_NORMAL)
101 		return 0;
102 
103 	if (capable(CAP_SYS_NICE))
104 		return 0;
105 
106 	if (drm_is_current_master(filp))
107 		return 0;
108 
109 	return -EACCES;
110 }
111 
112 static enum amdgpu_gfx_pipe_priority amdgpu_ctx_prio_to_compute_prio(int32_t prio)
113 {
114 	switch (prio) {
115 	case AMDGPU_CTX_PRIORITY_HIGH:
116 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
117 		return AMDGPU_GFX_PIPE_PRIO_HIGH;
118 	default:
119 		return AMDGPU_GFX_PIPE_PRIO_NORMAL;
120 	}
121 }
122 
123 static enum amdgpu_ring_priority_level amdgpu_ctx_sched_prio_to_ring_prio(int32_t prio)
124 {
125 	switch (prio) {
126 	case AMDGPU_CTX_PRIORITY_HIGH:
127 		return AMDGPU_RING_PRIO_1;
128 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
129 		return AMDGPU_RING_PRIO_2;
130 	default:
131 		return AMDGPU_RING_PRIO_0;
132 	}
133 }
134 
135 static unsigned int amdgpu_ctx_get_hw_prio(struct amdgpu_ctx *ctx, u32 hw_ip)
136 {
137 	struct amdgpu_device *adev = ctx->adev;
138 	int32_t ctx_prio;
139 	unsigned int hw_prio;
140 
141 	ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
142 			ctx->init_priority : ctx->override_priority;
143 
144 	switch (hw_ip) {
145 	case AMDGPU_HW_IP_COMPUTE:
146 		hw_prio = amdgpu_ctx_prio_to_compute_prio(ctx_prio);
147 		break;
148 	case AMDGPU_HW_IP_VCE:
149 	case AMDGPU_HW_IP_VCN_ENC:
150 		hw_prio = amdgpu_ctx_sched_prio_to_ring_prio(ctx_prio);
151 		break;
152 	default:
153 		hw_prio = AMDGPU_RING_PRIO_DEFAULT;
154 		break;
155 	}
156 
157 	hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
158 	if (adev->gpu_sched[hw_ip][hw_prio].num_scheds == 0)
159 		hw_prio = AMDGPU_RING_PRIO_DEFAULT;
160 
161 	return hw_prio;
162 }
163 
164 
165 static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip,
166 				  const u32 ring)
167 {
168 	struct amdgpu_device *adev = ctx->adev;
169 	struct amdgpu_ctx_entity *entity;
170 	struct drm_gpu_scheduler **scheds = NULL, *sched = NULL;
171 	unsigned num_scheds = 0;
172 	int32_t ctx_prio;
173 	unsigned int hw_prio;
174 	enum drm_sched_priority drm_prio;
175 	int r;
176 
177 	entity = kzalloc(struct_size(entity, fences, amdgpu_sched_jobs),
178 			 GFP_KERNEL);
179 	if (!entity)
180 		return  -ENOMEM;
181 
182 	ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
183 			ctx->init_priority : ctx->override_priority;
184 	entity->sequence = 1;
185 	hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip);
186 	drm_prio = amdgpu_ctx_to_drm_sched_prio(ctx_prio);
187 
188 	hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
189 	scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
190 	num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
191 
192 	/* disable load balance if the hw engine retains context among dependent jobs */
193 	if (hw_ip == AMDGPU_HW_IP_VCN_ENC ||
194 	    hw_ip == AMDGPU_HW_IP_VCN_DEC ||
195 	    hw_ip == AMDGPU_HW_IP_UVD_ENC ||
196 	    hw_ip == AMDGPU_HW_IP_UVD) {
197 		sched = drm_sched_pick_best(scheds, num_scheds);
198 		scheds = &sched;
199 		num_scheds = 1;
200 	}
201 
202 	r = drm_sched_entity_init(&entity->entity, drm_prio, scheds, num_scheds,
203 				  &ctx->guilty);
204 	if (r)
205 		goto error_free_entity;
206 
207 	ctx->entities[hw_ip][ring] = entity;
208 	return 0;
209 
210 error_free_entity:
211 	kfree(entity);
212 
213 	return r;
214 }
215 
216 static int amdgpu_ctx_init(struct amdgpu_device *adev,
217 			   int32_t priority,
218 			   struct drm_file *filp,
219 			   struct amdgpu_ctx *ctx)
220 {
221 	int r;
222 
223 	r = amdgpu_ctx_priority_permit(filp, priority);
224 	if (r)
225 		return r;
226 
227 	memset(ctx, 0, sizeof(*ctx));
228 
229 	ctx->adev = adev;
230 
231 	kref_init(&ctx->refcount);
232 	spin_lock_init(&ctx->ring_lock);
233 	mutex_init(&ctx->lock);
234 
235 	ctx->reset_counter = atomic_read(&adev->gpu_reset_counter);
236 	ctx->reset_counter_query = ctx->reset_counter;
237 	ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
238 	ctx->init_priority = priority;
239 	ctx->override_priority = AMDGPU_CTX_PRIORITY_UNSET;
240 
241 	return 0;
242 }
243 
244 static void amdgpu_ctx_fini_entity(struct amdgpu_ctx_entity *entity)
245 {
246 
247 	int i;
248 
249 	if (!entity)
250 		return;
251 
252 	for (i = 0; i < amdgpu_sched_jobs; ++i)
253 		dma_fence_put(entity->fences[i]);
254 
255 	kfree(entity);
256 }
257 
258 static void amdgpu_ctx_fini(struct kref *ref)
259 {
260 	struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount);
261 	struct amdgpu_device *adev = ctx->adev;
262 	unsigned i, j;
263 
264 	if (!adev)
265 		return;
266 
267 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
268 		for (j = 0; j < AMDGPU_MAX_ENTITY_NUM; ++j) {
269 			amdgpu_ctx_fini_entity(ctx->entities[i][j]);
270 			ctx->entities[i][j] = NULL;
271 		}
272 	}
273 
274 	mutex_destroy(&ctx->lock);
275 	kfree(ctx);
276 }
277 
278 int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance,
279 			  u32 ring, struct drm_sched_entity **entity)
280 {
281 	int r;
282 
283 	if (hw_ip >= AMDGPU_HW_IP_NUM) {
284 		DRM_ERROR("unknown HW IP type: %d\n", hw_ip);
285 		return -EINVAL;
286 	}
287 
288 	/* Right now all IPs have only one instance - multiple rings. */
289 	if (instance != 0) {
290 		DRM_DEBUG("invalid ip instance: %d\n", instance);
291 		return -EINVAL;
292 	}
293 
294 	if (ring >= amdgpu_ctx_num_entities[hw_ip]) {
295 		DRM_DEBUG("invalid ring: %d %d\n", hw_ip, ring);
296 		return -EINVAL;
297 	}
298 
299 	if (ctx->entities[hw_ip][ring] == NULL) {
300 		r = amdgpu_ctx_init_entity(ctx, hw_ip, ring);
301 		if (r)
302 			return r;
303 	}
304 
305 	*entity = &ctx->entities[hw_ip][ring]->entity;
306 	return 0;
307 }
308 
309 static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
310 			    struct amdgpu_fpriv *fpriv,
311 			    struct drm_file *filp,
312 			    int32_t priority,
313 			    uint32_t *id)
314 {
315 	struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
316 	struct amdgpu_ctx *ctx;
317 	int r;
318 
319 	ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
320 	if (!ctx)
321 		return -ENOMEM;
322 
323 	mutex_lock(&mgr->lock);
324 	r = idr_alloc(&mgr->ctx_handles, ctx, 1, AMDGPU_VM_MAX_NUM_CTX, GFP_KERNEL);
325 	if (r < 0) {
326 		mutex_unlock(&mgr->lock);
327 		kfree(ctx);
328 		return r;
329 	}
330 
331 	*id = (uint32_t)r;
332 	r = amdgpu_ctx_init(adev, priority, filp, ctx);
333 	if (r) {
334 		idr_remove(&mgr->ctx_handles, *id);
335 		*id = 0;
336 		kfree(ctx);
337 	}
338 	mutex_unlock(&mgr->lock);
339 	return r;
340 }
341 
342 static void amdgpu_ctx_do_release(struct kref *ref)
343 {
344 	struct amdgpu_ctx *ctx;
345 	u32 i, j;
346 
347 	ctx = container_of(ref, struct amdgpu_ctx, refcount);
348 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
349 		for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
350 			if (!ctx->entities[i][j])
351 				continue;
352 
353 			drm_sched_entity_destroy(&ctx->entities[i][j]->entity);
354 		}
355 	}
356 
357 	amdgpu_ctx_fini(ref);
358 }
359 
360 static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id)
361 {
362 	struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
363 	struct amdgpu_ctx *ctx;
364 
365 	mutex_lock(&mgr->lock);
366 	ctx = idr_remove(&mgr->ctx_handles, id);
367 	if (ctx)
368 		kref_put(&ctx->refcount, amdgpu_ctx_do_release);
369 	mutex_unlock(&mgr->lock);
370 	return ctx ? 0 : -EINVAL;
371 }
372 
373 static int amdgpu_ctx_query(struct amdgpu_device *adev,
374 			    struct amdgpu_fpriv *fpriv, uint32_t id,
375 			    union drm_amdgpu_ctx_out *out)
376 {
377 	struct amdgpu_ctx *ctx;
378 	struct amdgpu_ctx_mgr *mgr;
379 	unsigned reset_counter;
380 
381 	if (!fpriv)
382 		return -EINVAL;
383 
384 	mgr = &fpriv->ctx_mgr;
385 	mutex_lock(&mgr->lock);
386 	ctx = idr_find(&mgr->ctx_handles, id);
387 	if (!ctx) {
388 		mutex_unlock(&mgr->lock);
389 		return -EINVAL;
390 	}
391 
392 	/* TODO: these two are always zero */
393 	out->state.flags = 0x0;
394 	out->state.hangs = 0x0;
395 
396 	/* determine if a GPU reset has occured since the last call */
397 	reset_counter = atomic_read(&adev->gpu_reset_counter);
398 	/* TODO: this should ideally return NO, GUILTY, or INNOCENT. */
399 	if (ctx->reset_counter_query == reset_counter)
400 		out->state.reset_status = AMDGPU_CTX_NO_RESET;
401 	else
402 		out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET;
403 	ctx->reset_counter_query = reset_counter;
404 
405 	mutex_unlock(&mgr->lock);
406 	return 0;
407 }
408 
409 #define AMDGPU_RAS_COUNTE_DELAY_MS 3000
410 
411 static int amdgpu_ctx_query2(struct amdgpu_device *adev,
412 			     struct amdgpu_fpriv *fpriv, uint32_t id,
413 			     union drm_amdgpu_ctx_out *out)
414 {
415 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
416 	struct amdgpu_ctx *ctx;
417 	struct amdgpu_ctx_mgr *mgr;
418 
419 	if (!fpriv)
420 		return -EINVAL;
421 
422 	mgr = &fpriv->ctx_mgr;
423 	mutex_lock(&mgr->lock);
424 	ctx = idr_find(&mgr->ctx_handles, id);
425 	if (!ctx) {
426 		mutex_unlock(&mgr->lock);
427 		return -EINVAL;
428 	}
429 
430 	out->state.flags = 0x0;
431 	out->state.hangs = 0x0;
432 
433 	if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter))
434 		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET;
435 
436 	if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter))
437 		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST;
438 
439 	if (atomic_read(&ctx->guilty))
440 		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY;
441 
442 	if (adev->ras_enabled && con) {
443 		/* Return the cached values in O(1),
444 		 * and schedule delayed work to cache
445 		 * new vaues.
446 		 */
447 		int ce_count, ue_count;
448 
449 		ce_count = atomic_read(&con->ras_ce_count);
450 		ue_count = atomic_read(&con->ras_ue_count);
451 
452 		if (ce_count != ctx->ras_counter_ce) {
453 			ctx->ras_counter_ce = ce_count;
454 			out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_CE;
455 		}
456 
457 		if (ue_count != ctx->ras_counter_ue) {
458 			ctx->ras_counter_ue = ue_count;
459 			out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_UE;
460 		}
461 
462 		schedule_delayed_work(&con->ras_counte_delay_work,
463 				      msecs_to_jiffies(AMDGPU_RAS_COUNTE_DELAY_MS));
464 	}
465 
466 	mutex_unlock(&mgr->lock);
467 	return 0;
468 }
469 
470 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
471 		     struct drm_file *filp)
472 {
473 	int r;
474 	uint32_t id;
475 	int32_t priority;
476 
477 	union drm_amdgpu_ctx *args = data;
478 	struct amdgpu_device *adev = drm_to_adev(dev);
479 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
480 
481 	id = args->in.ctx_id;
482 	priority = args->in.priority;
483 
484 	/* For backwards compatibility reasons, we need to accept
485 	 * ioctls with garbage in the priority field */
486 	if (!amdgpu_ctx_priority_is_valid(priority))
487 		priority = AMDGPU_CTX_PRIORITY_NORMAL;
488 
489 	switch (args->in.op) {
490 	case AMDGPU_CTX_OP_ALLOC_CTX:
491 		r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id);
492 		args->out.alloc.ctx_id = id;
493 		break;
494 	case AMDGPU_CTX_OP_FREE_CTX:
495 		r = amdgpu_ctx_free(fpriv, id);
496 		break;
497 	case AMDGPU_CTX_OP_QUERY_STATE:
498 		r = amdgpu_ctx_query(adev, fpriv, id, &args->out);
499 		break;
500 	case AMDGPU_CTX_OP_QUERY_STATE2:
501 		r = amdgpu_ctx_query2(adev, fpriv, id, &args->out);
502 		break;
503 	default:
504 		return -EINVAL;
505 	}
506 
507 	return r;
508 }
509 
510 struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id)
511 {
512 	struct amdgpu_ctx *ctx;
513 	struct amdgpu_ctx_mgr *mgr;
514 
515 	if (!fpriv)
516 		return NULL;
517 
518 	mgr = &fpriv->ctx_mgr;
519 
520 	mutex_lock(&mgr->lock);
521 	ctx = idr_find(&mgr->ctx_handles, id);
522 	if (ctx)
523 		kref_get(&ctx->refcount);
524 	mutex_unlock(&mgr->lock);
525 	return ctx;
526 }
527 
528 int amdgpu_ctx_put(struct amdgpu_ctx *ctx)
529 {
530 	if (ctx == NULL)
531 		return -EINVAL;
532 
533 	kref_put(&ctx->refcount, amdgpu_ctx_do_release);
534 	return 0;
535 }
536 
537 void amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx,
538 			  struct drm_sched_entity *entity,
539 			  struct dma_fence *fence, uint64_t *handle)
540 {
541 	struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
542 	uint64_t seq = centity->sequence;
543 	struct dma_fence *other = NULL;
544 	unsigned idx = 0;
545 
546 	idx = seq & (amdgpu_sched_jobs - 1);
547 	other = centity->fences[idx];
548 	if (other)
549 		BUG_ON(!dma_fence_is_signaled(other));
550 
551 	dma_fence_get(fence);
552 
553 	spin_lock(&ctx->ring_lock);
554 	centity->fences[idx] = fence;
555 	centity->sequence++;
556 	spin_unlock(&ctx->ring_lock);
557 
558 	dma_fence_put(other);
559 	if (handle)
560 		*handle = seq;
561 }
562 
563 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
564 				       struct drm_sched_entity *entity,
565 				       uint64_t seq)
566 {
567 	struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
568 	struct dma_fence *fence;
569 
570 	spin_lock(&ctx->ring_lock);
571 
572 	if (seq == ~0ull)
573 		seq = centity->sequence - 1;
574 
575 	if (seq >= centity->sequence) {
576 		spin_unlock(&ctx->ring_lock);
577 		return ERR_PTR(-EINVAL);
578 	}
579 
580 
581 	if (seq + amdgpu_sched_jobs < centity->sequence) {
582 		spin_unlock(&ctx->ring_lock);
583 		return NULL;
584 	}
585 
586 	fence = dma_fence_get(centity->fences[seq & (amdgpu_sched_jobs - 1)]);
587 	spin_unlock(&ctx->ring_lock);
588 
589 	return fence;
590 }
591 
592 static void amdgpu_ctx_set_entity_priority(struct amdgpu_ctx *ctx,
593 					   struct amdgpu_ctx_entity *aentity,
594 					   int hw_ip,
595 					   int32_t priority)
596 {
597 	struct amdgpu_device *adev = ctx->adev;
598 	unsigned int hw_prio;
599 	struct drm_gpu_scheduler **scheds = NULL;
600 	unsigned num_scheds;
601 
602 	/* set sw priority */
603 	drm_sched_entity_set_priority(&aentity->entity,
604 				      amdgpu_ctx_to_drm_sched_prio(priority));
605 
606 	/* set hw priority */
607 	if (hw_ip == AMDGPU_HW_IP_COMPUTE) {
608 		hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip);
609 		hw_prio = array_index_nospec(hw_prio, AMDGPU_RING_PRIO_MAX);
610 		scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
611 		num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
612 		drm_sched_entity_modify_sched(&aentity->entity, scheds,
613 					      num_scheds);
614 	}
615 }
616 
617 void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
618 				  int32_t priority)
619 {
620 	int32_t ctx_prio;
621 	unsigned i, j;
622 
623 	ctx->override_priority = priority;
624 
625 	ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
626 			ctx->init_priority : ctx->override_priority;
627 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
628 		for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
629 			if (!ctx->entities[i][j])
630 				continue;
631 
632 			amdgpu_ctx_set_entity_priority(ctx, ctx->entities[i][j],
633 						       i, ctx_prio);
634 		}
635 	}
636 }
637 
638 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
639 			       struct drm_sched_entity *entity)
640 {
641 	struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
642 	struct dma_fence *other;
643 	unsigned idx;
644 	long r;
645 
646 	spin_lock(&ctx->ring_lock);
647 	idx = centity->sequence & (amdgpu_sched_jobs - 1);
648 	other = dma_fence_get(centity->fences[idx]);
649 	spin_unlock(&ctx->ring_lock);
650 
651 	if (!other)
652 		return 0;
653 
654 	r = dma_fence_wait(other, true);
655 	if (r < 0 && r != -ERESTARTSYS)
656 		DRM_ERROR("Error (%ld) waiting for fence!\n", r);
657 
658 	dma_fence_put(other);
659 	return r;
660 }
661 
662 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
663 {
664 	mutex_init(&mgr->lock);
665 	idr_init(&mgr->ctx_handles);
666 }
667 
668 long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout)
669 {
670 	struct amdgpu_ctx *ctx;
671 	struct idr *idp;
672 	uint32_t id, i, j;
673 
674 	idp = &mgr->ctx_handles;
675 
676 	mutex_lock(&mgr->lock);
677 	idr_for_each_entry(idp, ctx, id) {
678 		for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
679 			for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
680 				struct drm_sched_entity *entity;
681 
682 				if (!ctx->entities[i][j])
683 					continue;
684 
685 				entity = &ctx->entities[i][j]->entity;
686 				timeout = drm_sched_entity_flush(entity, timeout);
687 			}
688 		}
689 	}
690 	mutex_unlock(&mgr->lock);
691 	return timeout;
692 }
693 
694 void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr)
695 {
696 	struct amdgpu_ctx *ctx;
697 	struct idr *idp;
698 	uint32_t id, i, j;
699 
700 	idp = &mgr->ctx_handles;
701 
702 	idr_for_each_entry(idp, ctx, id) {
703 		if (kref_read(&ctx->refcount) != 1) {
704 			DRM_ERROR("ctx %p is still alive\n", ctx);
705 			continue;
706 		}
707 
708 		for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
709 			for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
710 				struct drm_sched_entity *entity;
711 
712 				if (!ctx->entities[i][j])
713 					continue;
714 
715 				entity = &ctx->entities[i][j]->entity;
716 				drm_sched_entity_fini(entity);
717 			}
718 		}
719 	}
720 }
721 
722 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
723 {
724 	struct amdgpu_ctx *ctx;
725 	struct idr *idp;
726 	uint32_t id;
727 
728 	amdgpu_ctx_mgr_entity_fini(mgr);
729 
730 	idp = &mgr->ctx_handles;
731 
732 	idr_for_each_entry(idp, ctx, id) {
733 		if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1)
734 			DRM_ERROR("ctx %p is still alive\n", ctx);
735 	}
736 
737 	idr_destroy(&mgr->ctx_handles);
738 	mutex_destroy(&mgr->lock);
739 }
740 
741 static void amdgpu_ctx_fence_time(struct amdgpu_ctx *ctx,
742 		struct amdgpu_ctx_entity *centity, ktime_t *total, ktime_t *max)
743 {
744 	ktime_t now, t1;
745 	uint32_t i;
746 
747 	*total = *max = 0;
748 
749 	now = ktime_get();
750 	for (i = 0; i < amdgpu_sched_jobs; i++) {
751 		struct dma_fence *fence;
752 		struct drm_sched_fence *s_fence;
753 
754 		spin_lock(&ctx->ring_lock);
755 		fence = dma_fence_get(centity->fences[i]);
756 		spin_unlock(&ctx->ring_lock);
757 		if (!fence)
758 			continue;
759 		s_fence = to_drm_sched_fence(fence);
760 		if (!dma_fence_is_signaled(&s_fence->scheduled)) {
761 			dma_fence_put(fence);
762 			continue;
763 		}
764 		t1 = s_fence->scheduled.timestamp;
765 		if (!ktime_before(t1, now)) {
766 			dma_fence_put(fence);
767 			continue;
768 		}
769 		if (dma_fence_is_signaled(&s_fence->finished) &&
770 			s_fence->finished.timestamp < now)
771 			*total += ktime_sub(s_fence->finished.timestamp, t1);
772 		else
773 			*total += ktime_sub(now, t1);
774 		t1 = ktime_sub(now, t1);
775 		dma_fence_put(fence);
776 		*max = max(t1, *max);
777 	}
778 }
779 
780 ktime_t amdgpu_ctx_mgr_fence_usage(struct amdgpu_ctx_mgr *mgr, uint32_t hwip,
781 		uint32_t idx, uint64_t *elapsed)
782 {
783 	struct idr *idp;
784 	struct amdgpu_ctx *ctx;
785 	uint32_t id;
786 	struct amdgpu_ctx_entity *centity;
787 	ktime_t total = 0, max = 0;
788 
789 	if (idx >= AMDGPU_MAX_ENTITY_NUM)
790 		return 0;
791 	idp = &mgr->ctx_handles;
792 	mutex_lock(&mgr->lock);
793 	idr_for_each_entry(idp, ctx, id) {
794 		ktime_t ttotal, tmax;
795 
796 		if (!ctx->entities[hwip][idx])
797 			continue;
798 
799 		centity = ctx->entities[hwip][idx];
800 		amdgpu_ctx_fence_time(ctx, centity, &ttotal, &tmax);
801 
802 		/* Harmonic mean approximation diverges for very small
803 		 * values. If ratio < 0.01% ignore
804 		 */
805 		if (AMDGPU_CTX_FENCE_USAGE_MIN_RATIO(tmax, ttotal))
806 			continue;
807 
808 		total = ktime_add(total, ttotal);
809 		max = ktime_after(tmax, max) ? tmax : max;
810 	}
811 
812 	mutex_unlock(&mgr->lock);
813 	if (elapsed)
814 		*elapsed = max;
815 
816 	return total;
817 }
818