1 /*
2  * Copyright 2015 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: monk liu <monk.liu@amd.com>
23  */
24 
25 #include <drm/drm_auth.h>
26 #include <drm/drm_drv.h>
27 #include "amdgpu.h"
28 #include "amdgpu_sched.h"
29 #include "amdgpu_ras.h"
30 #include <linux/nospec.h>
31 
32 #define to_amdgpu_ctx_entity(e)	\
33 	container_of((e), struct amdgpu_ctx_entity, entity)
34 
35 const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM] = {
36 	[AMDGPU_HW_IP_GFX]	=	1,
37 	[AMDGPU_HW_IP_COMPUTE]	=	4,
38 	[AMDGPU_HW_IP_DMA]	=	2,
39 	[AMDGPU_HW_IP_UVD]	=	1,
40 	[AMDGPU_HW_IP_VCE]	=	1,
41 	[AMDGPU_HW_IP_UVD_ENC]	=	1,
42 	[AMDGPU_HW_IP_VCN_DEC]	=	1,
43 	[AMDGPU_HW_IP_VCN_ENC]	=	1,
44 	[AMDGPU_HW_IP_VCN_JPEG]	=	1,
45 };
46 
47 bool amdgpu_ctx_priority_is_valid(int32_t ctx_prio)
48 {
49 	switch (ctx_prio) {
50 	case AMDGPU_CTX_PRIORITY_UNSET:
51 	case AMDGPU_CTX_PRIORITY_VERY_LOW:
52 	case AMDGPU_CTX_PRIORITY_LOW:
53 	case AMDGPU_CTX_PRIORITY_NORMAL:
54 	case AMDGPU_CTX_PRIORITY_HIGH:
55 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
56 		return true;
57 	default:
58 		return false;
59 	}
60 }
61 
62 static enum drm_sched_priority
63 amdgpu_ctx_to_drm_sched_prio(int32_t ctx_prio)
64 {
65 	switch (ctx_prio) {
66 	case AMDGPU_CTX_PRIORITY_UNSET:
67 		return DRM_SCHED_PRIORITY_UNSET;
68 
69 	case AMDGPU_CTX_PRIORITY_VERY_LOW:
70 		return DRM_SCHED_PRIORITY_MIN;
71 
72 	case AMDGPU_CTX_PRIORITY_LOW:
73 		return DRM_SCHED_PRIORITY_MIN;
74 
75 	case AMDGPU_CTX_PRIORITY_NORMAL:
76 		return DRM_SCHED_PRIORITY_NORMAL;
77 
78 	case AMDGPU_CTX_PRIORITY_HIGH:
79 		return DRM_SCHED_PRIORITY_HIGH;
80 
81 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
82 		return DRM_SCHED_PRIORITY_HIGH;
83 
84 	/* This should not happen as we sanitized userspace provided priority
85 	 * already, WARN if this happens.
86 	 */
87 	default:
88 		WARN(1, "Invalid context priority %d\n", ctx_prio);
89 		return DRM_SCHED_PRIORITY_NORMAL;
90 	}
91 
92 }
93 
94 static int amdgpu_ctx_priority_permit(struct drm_file *filp,
95 				      int32_t priority)
96 {
97 	if (!amdgpu_ctx_priority_is_valid(priority))
98 		return -EINVAL;
99 
100 	/* NORMAL and below are accessible by everyone */
101 	if (priority <= AMDGPU_CTX_PRIORITY_NORMAL)
102 		return 0;
103 
104 	if (capable(CAP_SYS_NICE))
105 		return 0;
106 
107 	if (drm_is_current_master(filp))
108 		return 0;
109 
110 	return -EACCES;
111 }
112 
113 static enum amdgpu_gfx_pipe_priority amdgpu_ctx_prio_to_gfx_pipe_prio(int32_t prio)
114 {
115 	switch (prio) {
116 	case AMDGPU_CTX_PRIORITY_HIGH:
117 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
118 		return AMDGPU_GFX_PIPE_PRIO_HIGH;
119 	default:
120 		return AMDGPU_GFX_PIPE_PRIO_NORMAL;
121 	}
122 }
123 
124 static enum amdgpu_ring_priority_level amdgpu_ctx_sched_prio_to_ring_prio(int32_t prio)
125 {
126 	switch (prio) {
127 	case AMDGPU_CTX_PRIORITY_HIGH:
128 		return AMDGPU_RING_PRIO_1;
129 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
130 		return AMDGPU_RING_PRIO_2;
131 	default:
132 		return AMDGPU_RING_PRIO_0;
133 	}
134 }
135 
136 static unsigned int amdgpu_ctx_get_hw_prio(struct amdgpu_ctx *ctx, u32 hw_ip)
137 {
138 	struct amdgpu_device *adev = ctx->mgr->adev;
139 	unsigned int hw_prio;
140 	int32_t ctx_prio;
141 
142 	ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
143 			ctx->init_priority : ctx->override_priority;
144 
145 	switch (hw_ip) {
146 	case AMDGPU_HW_IP_GFX:
147 	case AMDGPU_HW_IP_COMPUTE:
148 		hw_prio = amdgpu_ctx_prio_to_gfx_pipe_prio(ctx_prio);
149 		break;
150 	case AMDGPU_HW_IP_VCE:
151 	case AMDGPU_HW_IP_VCN_ENC:
152 		hw_prio = amdgpu_ctx_sched_prio_to_ring_prio(ctx_prio);
153 		break;
154 	default:
155 		hw_prio = AMDGPU_RING_PRIO_DEFAULT;
156 		break;
157 	}
158 
159 	hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
160 	if (adev->gpu_sched[hw_ip][hw_prio].num_scheds == 0)
161 		hw_prio = AMDGPU_RING_PRIO_DEFAULT;
162 
163 	return hw_prio;
164 }
165 
166 /* Calculate the time spend on the hw */
167 static ktime_t amdgpu_ctx_fence_time(struct dma_fence *fence)
168 {
169 	struct drm_sched_fence *s_fence;
170 
171 	if (!fence)
172 		return ns_to_ktime(0);
173 
174 	/* When the fence is not even scheduled it can't have spend time */
175 	s_fence = to_drm_sched_fence(fence);
176 	if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &s_fence->scheduled.flags))
177 		return ns_to_ktime(0);
178 
179 	/* When it is still running account how much already spend */
180 	if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &s_fence->finished.flags))
181 		return ktime_sub(ktime_get(), s_fence->scheduled.timestamp);
182 
183 	return ktime_sub(s_fence->finished.timestamp,
184 			 s_fence->scheduled.timestamp);
185 }
186 
187 static ktime_t amdgpu_ctx_entity_time(struct amdgpu_ctx *ctx,
188 				      struct amdgpu_ctx_entity *centity)
189 {
190 	ktime_t res = ns_to_ktime(0);
191 	uint32_t i;
192 
193 	spin_lock(&ctx->ring_lock);
194 	for (i = 0; i < amdgpu_sched_jobs; i++) {
195 		res = ktime_add(res, amdgpu_ctx_fence_time(centity->fences[i]));
196 	}
197 	spin_unlock(&ctx->ring_lock);
198 	return res;
199 }
200 
201 static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip,
202 				  const u32 ring)
203 {
204 	struct drm_gpu_scheduler **scheds = NULL, *sched = NULL;
205 	struct amdgpu_device *adev = ctx->mgr->adev;
206 	struct amdgpu_ctx_entity *entity;
207 	enum drm_sched_priority drm_prio;
208 	unsigned int hw_prio, num_scheds;
209 	int32_t ctx_prio;
210 	int r;
211 
212 	entity = kzalloc(struct_size(entity, fences, amdgpu_sched_jobs),
213 			 GFP_KERNEL);
214 	if (!entity)
215 		return  -ENOMEM;
216 
217 	ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
218 			ctx->init_priority : ctx->override_priority;
219 	entity->hw_ip = hw_ip;
220 	entity->sequence = 1;
221 	hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip);
222 	drm_prio = amdgpu_ctx_to_drm_sched_prio(ctx_prio);
223 
224 	hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
225 	scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
226 	num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
227 
228 	/* disable load balance if the hw engine retains context among dependent jobs */
229 	if (hw_ip == AMDGPU_HW_IP_VCN_ENC ||
230 	    hw_ip == AMDGPU_HW_IP_VCN_DEC ||
231 	    hw_ip == AMDGPU_HW_IP_UVD_ENC ||
232 	    hw_ip == AMDGPU_HW_IP_UVD) {
233 		sched = drm_sched_pick_best(scheds, num_scheds);
234 		scheds = &sched;
235 		num_scheds = 1;
236 	}
237 
238 	r = drm_sched_entity_init(&entity->entity, drm_prio, scheds, num_scheds,
239 				  &ctx->guilty);
240 	if (r)
241 		goto error_free_entity;
242 
243 	/* It's not an error if we fail to install the new entity */
244 	if (cmpxchg(&ctx->entities[hw_ip][ring], NULL, entity))
245 		goto cleanup_entity;
246 
247 	return 0;
248 
249 cleanup_entity:
250 	drm_sched_entity_fini(&entity->entity);
251 
252 error_free_entity:
253 	kfree(entity);
254 
255 	return r;
256 }
257 
258 static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_ctx_entity *entity)
259 {
260 	ktime_t res = ns_to_ktime(0);
261 	int i;
262 
263 	if (!entity)
264 		return res;
265 
266 	for (i = 0; i < amdgpu_sched_jobs; ++i) {
267 		res = ktime_add(res, amdgpu_ctx_fence_time(entity->fences[i]));
268 		dma_fence_put(entity->fences[i]);
269 	}
270 
271 	kfree(entity);
272 	return res;
273 }
274 
275 static int amdgpu_ctx_get_stable_pstate(struct amdgpu_ctx *ctx,
276 					u32 *stable_pstate)
277 {
278 	struct amdgpu_device *adev = ctx->mgr->adev;
279 	enum amd_dpm_forced_level current_level;
280 
281 	current_level = amdgpu_dpm_get_performance_level(adev);
282 
283 	switch (current_level) {
284 	case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
285 		*stable_pstate = AMDGPU_CTX_STABLE_PSTATE_STANDARD;
286 		break;
287 	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
288 		*stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK;
289 		break;
290 	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
291 		*stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK;
292 		break;
293 	case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
294 		*stable_pstate = AMDGPU_CTX_STABLE_PSTATE_PEAK;
295 		break;
296 	default:
297 		*stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE;
298 		break;
299 	}
300 	return 0;
301 }
302 
303 static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority,
304 			   struct drm_file *filp, struct amdgpu_ctx *ctx)
305 {
306 	u32 current_stable_pstate;
307 	int r;
308 
309 	r = amdgpu_ctx_priority_permit(filp, priority);
310 	if (r)
311 		return r;
312 
313 	memset(ctx, 0, sizeof(*ctx));
314 
315 	kref_init(&ctx->refcount);
316 	ctx->mgr = mgr;
317 	spin_lock_init(&ctx->ring_lock);
318 	mutex_init(&ctx->lock);
319 
320 	ctx->reset_counter = atomic_read(&mgr->adev->gpu_reset_counter);
321 	ctx->reset_counter_query = ctx->reset_counter;
322 	ctx->vram_lost_counter = atomic_read(&mgr->adev->vram_lost_counter);
323 	ctx->init_priority = priority;
324 	ctx->override_priority = AMDGPU_CTX_PRIORITY_UNSET;
325 
326 	r = amdgpu_ctx_get_stable_pstate(ctx, &current_stable_pstate);
327 	if (r)
328 		return r;
329 
330 	ctx->stable_pstate = current_stable_pstate;
331 
332 	return 0;
333 }
334 
335 static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx,
336 					u32 stable_pstate)
337 {
338 	struct amdgpu_device *adev = ctx->mgr->adev;
339 	enum amd_dpm_forced_level level;
340 	u32 current_stable_pstate;
341 	int r;
342 
343 	mutex_lock(&adev->pm.stable_pstate_ctx_lock);
344 	if (adev->pm.stable_pstate_ctx && adev->pm.stable_pstate_ctx != ctx) {
345 		r = -EBUSY;
346 		goto done;
347 	}
348 
349 	r = amdgpu_ctx_get_stable_pstate(ctx, &current_stable_pstate);
350 	if (r || (stable_pstate == current_stable_pstate))
351 		goto done;
352 
353 	switch (stable_pstate) {
354 	case AMDGPU_CTX_STABLE_PSTATE_NONE:
355 		level = AMD_DPM_FORCED_LEVEL_AUTO;
356 		break;
357 	case AMDGPU_CTX_STABLE_PSTATE_STANDARD:
358 		level = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD;
359 		break;
360 	case AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK:
361 		level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK;
362 		break;
363 	case AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK:
364 		level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK;
365 		break;
366 	case AMDGPU_CTX_STABLE_PSTATE_PEAK:
367 		level = AMD_DPM_FORCED_LEVEL_PROFILE_PEAK;
368 		break;
369 	default:
370 		r = -EINVAL;
371 		goto done;
372 	}
373 
374 	r = amdgpu_dpm_force_performance_level(adev, level);
375 
376 	if (level == AMD_DPM_FORCED_LEVEL_AUTO)
377 		adev->pm.stable_pstate_ctx = NULL;
378 	else
379 		adev->pm.stable_pstate_ctx = ctx;
380 done:
381 	mutex_unlock(&adev->pm.stable_pstate_ctx_lock);
382 
383 	return r;
384 }
385 
386 static void amdgpu_ctx_fini(struct kref *ref)
387 {
388 	struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount);
389 	struct amdgpu_ctx_mgr *mgr = ctx->mgr;
390 	struct amdgpu_device *adev = mgr->adev;
391 	unsigned i, j, idx;
392 
393 	if (!adev)
394 		return;
395 
396 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
397 		for (j = 0; j < AMDGPU_MAX_ENTITY_NUM; ++j) {
398 			ktime_t spend;
399 
400 			spend = amdgpu_ctx_fini_entity(ctx->entities[i][j]);
401 			atomic64_add(ktime_to_ns(spend), &mgr->time_spend[i]);
402 		}
403 	}
404 
405 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
406 		amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate);
407 		drm_dev_exit(idx);
408 	}
409 
410 	mutex_destroy(&ctx->lock);
411 	kfree(ctx);
412 }
413 
414 int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance,
415 			  u32 ring, struct drm_sched_entity **entity)
416 {
417 	int r;
418 
419 	if (hw_ip >= AMDGPU_HW_IP_NUM) {
420 		DRM_ERROR("unknown HW IP type: %d\n", hw_ip);
421 		return -EINVAL;
422 	}
423 
424 	/* Right now all IPs have only one instance - multiple rings. */
425 	if (instance != 0) {
426 		DRM_DEBUG("invalid ip instance: %d\n", instance);
427 		return -EINVAL;
428 	}
429 
430 	if (ring >= amdgpu_ctx_num_entities[hw_ip]) {
431 		DRM_DEBUG("invalid ring: %d %d\n", hw_ip, ring);
432 		return -EINVAL;
433 	}
434 
435 	if (ctx->entities[hw_ip][ring] == NULL) {
436 		r = amdgpu_ctx_init_entity(ctx, hw_ip, ring);
437 		if (r)
438 			return r;
439 	}
440 
441 	*entity = &ctx->entities[hw_ip][ring]->entity;
442 	return 0;
443 }
444 
445 static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
446 			    struct amdgpu_fpriv *fpriv,
447 			    struct drm_file *filp,
448 			    int32_t priority,
449 			    uint32_t *id)
450 {
451 	struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
452 	struct amdgpu_ctx *ctx;
453 	int r;
454 
455 	ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
456 	if (!ctx)
457 		return -ENOMEM;
458 
459 	mutex_lock(&mgr->lock);
460 	r = idr_alloc(&mgr->ctx_handles, ctx, 1, AMDGPU_VM_MAX_NUM_CTX, GFP_KERNEL);
461 	if (r < 0) {
462 		mutex_unlock(&mgr->lock);
463 		kfree(ctx);
464 		return r;
465 	}
466 
467 	*id = (uint32_t)r;
468 	r = amdgpu_ctx_init(mgr, priority, filp, ctx);
469 	if (r) {
470 		idr_remove(&mgr->ctx_handles, *id);
471 		*id = 0;
472 		kfree(ctx);
473 	}
474 	mutex_unlock(&mgr->lock);
475 	return r;
476 }
477 
478 static void amdgpu_ctx_do_release(struct kref *ref)
479 {
480 	struct amdgpu_ctx *ctx;
481 	u32 i, j;
482 
483 	ctx = container_of(ref, struct amdgpu_ctx, refcount);
484 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
485 		for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
486 			if (!ctx->entities[i][j])
487 				continue;
488 
489 			drm_sched_entity_destroy(&ctx->entities[i][j]->entity);
490 		}
491 	}
492 
493 	amdgpu_ctx_fini(ref);
494 }
495 
496 static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id)
497 {
498 	struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
499 	struct amdgpu_ctx *ctx;
500 
501 	mutex_lock(&mgr->lock);
502 	ctx = idr_remove(&mgr->ctx_handles, id);
503 	if (ctx)
504 		kref_put(&ctx->refcount, amdgpu_ctx_do_release);
505 	mutex_unlock(&mgr->lock);
506 	return ctx ? 0 : -EINVAL;
507 }
508 
509 static int amdgpu_ctx_query(struct amdgpu_device *adev,
510 			    struct amdgpu_fpriv *fpriv, uint32_t id,
511 			    union drm_amdgpu_ctx_out *out)
512 {
513 	struct amdgpu_ctx *ctx;
514 	struct amdgpu_ctx_mgr *mgr;
515 	unsigned reset_counter;
516 
517 	if (!fpriv)
518 		return -EINVAL;
519 
520 	mgr = &fpriv->ctx_mgr;
521 	mutex_lock(&mgr->lock);
522 	ctx = idr_find(&mgr->ctx_handles, id);
523 	if (!ctx) {
524 		mutex_unlock(&mgr->lock);
525 		return -EINVAL;
526 	}
527 
528 	/* TODO: these two are always zero */
529 	out->state.flags = 0x0;
530 	out->state.hangs = 0x0;
531 
532 	/* determine if a GPU reset has occured since the last call */
533 	reset_counter = atomic_read(&adev->gpu_reset_counter);
534 	/* TODO: this should ideally return NO, GUILTY, or INNOCENT. */
535 	if (ctx->reset_counter_query == reset_counter)
536 		out->state.reset_status = AMDGPU_CTX_NO_RESET;
537 	else
538 		out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET;
539 	ctx->reset_counter_query = reset_counter;
540 
541 	mutex_unlock(&mgr->lock);
542 	return 0;
543 }
544 
545 #define AMDGPU_RAS_COUNTE_DELAY_MS 3000
546 
547 static int amdgpu_ctx_query2(struct amdgpu_device *adev,
548 			     struct amdgpu_fpriv *fpriv, uint32_t id,
549 			     union drm_amdgpu_ctx_out *out)
550 {
551 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
552 	struct amdgpu_ctx *ctx;
553 	struct amdgpu_ctx_mgr *mgr;
554 
555 	if (!fpriv)
556 		return -EINVAL;
557 
558 	mgr = &fpriv->ctx_mgr;
559 	mutex_lock(&mgr->lock);
560 	ctx = idr_find(&mgr->ctx_handles, id);
561 	if (!ctx) {
562 		mutex_unlock(&mgr->lock);
563 		return -EINVAL;
564 	}
565 
566 	out->state.flags = 0x0;
567 	out->state.hangs = 0x0;
568 
569 	if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter))
570 		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET;
571 
572 	if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter))
573 		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST;
574 
575 	if (atomic_read(&ctx->guilty))
576 		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY;
577 
578 	if (adev->ras_enabled && con) {
579 		/* Return the cached values in O(1),
580 		 * and schedule delayed work to cache
581 		 * new vaues.
582 		 */
583 		int ce_count, ue_count;
584 
585 		ce_count = atomic_read(&con->ras_ce_count);
586 		ue_count = atomic_read(&con->ras_ue_count);
587 
588 		if (ce_count != ctx->ras_counter_ce) {
589 			ctx->ras_counter_ce = ce_count;
590 			out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_CE;
591 		}
592 
593 		if (ue_count != ctx->ras_counter_ue) {
594 			ctx->ras_counter_ue = ue_count;
595 			out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_UE;
596 		}
597 
598 		schedule_delayed_work(&con->ras_counte_delay_work,
599 				      msecs_to_jiffies(AMDGPU_RAS_COUNTE_DELAY_MS));
600 	}
601 
602 	mutex_unlock(&mgr->lock);
603 	return 0;
604 }
605 
606 
607 
608 static int amdgpu_ctx_stable_pstate(struct amdgpu_device *adev,
609 				    struct amdgpu_fpriv *fpriv, uint32_t id,
610 				    bool set, u32 *stable_pstate)
611 {
612 	struct amdgpu_ctx *ctx;
613 	struct amdgpu_ctx_mgr *mgr;
614 	int r;
615 
616 	if (!fpriv)
617 		return -EINVAL;
618 
619 	mgr = &fpriv->ctx_mgr;
620 	mutex_lock(&mgr->lock);
621 	ctx = idr_find(&mgr->ctx_handles, id);
622 	if (!ctx) {
623 		mutex_unlock(&mgr->lock);
624 		return -EINVAL;
625 	}
626 
627 	if (set)
628 		r = amdgpu_ctx_set_stable_pstate(ctx, *stable_pstate);
629 	else
630 		r = amdgpu_ctx_get_stable_pstate(ctx, stable_pstate);
631 
632 	mutex_unlock(&mgr->lock);
633 	return r;
634 }
635 
636 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
637 		     struct drm_file *filp)
638 {
639 	int r;
640 	uint32_t id, stable_pstate;
641 	int32_t priority;
642 
643 	union drm_amdgpu_ctx *args = data;
644 	struct amdgpu_device *adev = drm_to_adev(dev);
645 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
646 
647 	id = args->in.ctx_id;
648 	priority = args->in.priority;
649 
650 	/* For backwards compatibility reasons, we need to accept
651 	 * ioctls with garbage in the priority field */
652 	if (!amdgpu_ctx_priority_is_valid(priority))
653 		priority = AMDGPU_CTX_PRIORITY_NORMAL;
654 
655 	switch (args->in.op) {
656 	case AMDGPU_CTX_OP_ALLOC_CTX:
657 		r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id);
658 		args->out.alloc.ctx_id = id;
659 		break;
660 	case AMDGPU_CTX_OP_FREE_CTX:
661 		r = amdgpu_ctx_free(fpriv, id);
662 		break;
663 	case AMDGPU_CTX_OP_QUERY_STATE:
664 		r = amdgpu_ctx_query(adev, fpriv, id, &args->out);
665 		break;
666 	case AMDGPU_CTX_OP_QUERY_STATE2:
667 		r = amdgpu_ctx_query2(adev, fpriv, id, &args->out);
668 		break;
669 	case AMDGPU_CTX_OP_GET_STABLE_PSTATE:
670 		if (args->in.flags)
671 			return -EINVAL;
672 		r = amdgpu_ctx_stable_pstate(adev, fpriv, id, false, &stable_pstate);
673 		if (!r)
674 			args->out.pstate.flags = stable_pstate;
675 		break;
676 	case AMDGPU_CTX_OP_SET_STABLE_PSTATE:
677 		if (args->in.flags & ~AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK)
678 			return -EINVAL;
679 		stable_pstate = args->in.flags & AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK;
680 		if (stable_pstate > AMDGPU_CTX_STABLE_PSTATE_PEAK)
681 			return -EINVAL;
682 		r = amdgpu_ctx_stable_pstate(adev, fpriv, id, true, &stable_pstate);
683 		break;
684 	default:
685 		return -EINVAL;
686 	}
687 
688 	return r;
689 }
690 
691 struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id)
692 {
693 	struct amdgpu_ctx *ctx;
694 	struct amdgpu_ctx_mgr *mgr;
695 
696 	if (!fpriv)
697 		return NULL;
698 
699 	mgr = &fpriv->ctx_mgr;
700 
701 	mutex_lock(&mgr->lock);
702 	ctx = idr_find(&mgr->ctx_handles, id);
703 	if (ctx)
704 		kref_get(&ctx->refcount);
705 	mutex_unlock(&mgr->lock);
706 	return ctx;
707 }
708 
709 int amdgpu_ctx_put(struct amdgpu_ctx *ctx)
710 {
711 	if (ctx == NULL)
712 		return -EINVAL;
713 
714 	kref_put(&ctx->refcount, amdgpu_ctx_do_release);
715 	return 0;
716 }
717 
718 uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx,
719 			      struct drm_sched_entity *entity,
720 			      struct dma_fence *fence)
721 {
722 	struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
723 	uint64_t seq = centity->sequence;
724 	struct dma_fence *other = NULL;
725 	unsigned idx = 0;
726 
727 	idx = seq & (amdgpu_sched_jobs - 1);
728 	other = centity->fences[idx];
729 	WARN_ON(other && !dma_fence_is_signaled(other));
730 
731 	dma_fence_get(fence);
732 
733 	spin_lock(&ctx->ring_lock);
734 	centity->fences[idx] = fence;
735 	centity->sequence++;
736 	spin_unlock(&ctx->ring_lock);
737 
738 	atomic64_add(ktime_to_ns(amdgpu_ctx_fence_time(other)),
739 		     &ctx->mgr->time_spend[centity->hw_ip]);
740 
741 	dma_fence_put(other);
742 	return seq;
743 }
744 
745 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
746 				       struct drm_sched_entity *entity,
747 				       uint64_t seq)
748 {
749 	struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
750 	struct dma_fence *fence;
751 
752 	spin_lock(&ctx->ring_lock);
753 
754 	if (seq == ~0ull)
755 		seq = centity->sequence - 1;
756 
757 	if (seq >= centity->sequence) {
758 		spin_unlock(&ctx->ring_lock);
759 		return ERR_PTR(-EINVAL);
760 	}
761 
762 
763 	if (seq + amdgpu_sched_jobs < centity->sequence) {
764 		spin_unlock(&ctx->ring_lock);
765 		return NULL;
766 	}
767 
768 	fence = dma_fence_get(centity->fences[seq & (amdgpu_sched_jobs - 1)]);
769 	spin_unlock(&ctx->ring_lock);
770 
771 	return fence;
772 }
773 
774 static void amdgpu_ctx_set_entity_priority(struct amdgpu_ctx *ctx,
775 					   struct amdgpu_ctx_entity *aentity,
776 					   int hw_ip,
777 					   int32_t priority)
778 {
779 	struct amdgpu_device *adev = ctx->mgr->adev;
780 	unsigned int hw_prio;
781 	struct drm_gpu_scheduler **scheds = NULL;
782 	unsigned num_scheds;
783 
784 	/* set sw priority */
785 	drm_sched_entity_set_priority(&aentity->entity,
786 				      amdgpu_ctx_to_drm_sched_prio(priority));
787 
788 	/* set hw priority */
789 	if (hw_ip == AMDGPU_HW_IP_COMPUTE || hw_ip == AMDGPU_HW_IP_GFX) {
790 		hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip);
791 		hw_prio = array_index_nospec(hw_prio, AMDGPU_RING_PRIO_MAX);
792 		scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
793 		num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
794 		drm_sched_entity_modify_sched(&aentity->entity, scheds,
795 					      num_scheds);
796 	}
797 }
798 
799 void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
800 				  int32_t priority)
801 {
802 	int32_t ctx_prio;
803 	unsigned i, j;
804 
805 	ctx->override_priority = priority;
806 
807 	ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
808 			ctx->init_priority : ctx->override_priority;
809 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
810 		for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
811 			if (!ctx->entities[i][j])
812 				continue;
813 
814 			amdgpu_ctx_set_entity_priority(ctx, ctx->entities[i][j],
815 						       i, ctx_prio);
816 		}
817 	}
818 }
819 
820 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
821 			       struct drm_sched_entity *entity)
822 {
823 	struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
824 	struct dma_fence *other;
825 	unsigned idx;
826 	long r;
827 
828 	spin_lock(&ctx->ring_lock);
829 	idx = centity->sequence & (amdgpu_sched_jobs - 1);
830 	other = dma_fence_get(centity->fences[idx]);
831 	spin_unlock(&ctx->ring_lock);
832 
833 	if (!other)
834 		return 0;
835 
836 	r = dma_fence_wait(other, true);
837 	if (r < 0 && r != -ERESTARTSYS)
838 		DRM_ERROR("Error (%ld) waiting for fence!\n", r);
839 
840 	dma_fence_put(other);
841 	return r;
842 }
843 
844 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr,
845 			 struct amdgpu_device *adev)
846 {
847 	unsigned int i;
848 
849 	mgr->adev = adev;
850 	mutex_init(&mgr->lock);
851 	idr_init_base(&mgr->ctx_handles, 1);
852 
853 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
854 		atomic64_set(&mgr->time_spend[i], 0);
855 }
856 
857 long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout)
858 {
859 	struct amdgpu_ctx *ctx;
860 	struct idr *idp;
861 	uint32_t id, i, j;
862 
863 	idp = &mgr->ctx_handles;
864 
865 	mutex_lock(&mgr->lock);
866 	idr_for_each_entry(idp, ctx, id) {
867 		for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
868 			for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
869 				struct drm_sched_entity *entity;
870 
871 				if (!ctx->entities[i][j])
872 					continue;
873 
874 				entity = &ctx->entities[i][j]->entity;
875 				timeout = drm_sched_entity_flush(entity, timeout);
876 			}
877 		}
878 	}
879 	mutex_unlock(&mgr->lock);
880 	return timeout;
881 }
882 
883 void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr)
884 {
885 	struct amdgpu_ctx *ctx;
886 	struct idr *idp;
887 	uint32_t id, i, j;
888 
889 	idp = &mgr->ctx_handles;
890 
891 	idr_for_each_entry(idp, ctx, id) {
892 		if (kref_read(&ctx->refcount) != 1) {
893 			DRM_ERROR("ctx %p is still alive\n", ctx);
894 			continue;
895 		}
896 
897 		for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
898 			for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
899 				struct drm_sched_entity *entity;
900 
901 				if (!ctx->entities[i][j])
902 					continue;
903 
904 				entity = &ctx->entities[i][j]->entity;
905 				drm_sched_entity_fini(entity);
906 			}
907 		}
908 	}
909 }
910 
911 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
912 {
913 	struct amdgpu_ctx *ctx;
914 	struct idr *idp;
915 	uint32_t id;
916 
917 	amdgpu_ctx_mgr_entity_fini(mgr);
918 
919 	idp = &mgr->ctx_handles;
920 
921 	idr_for_each_entry(idp, ctx, id) {
922 		if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1)
923 			DRM_ERROR("ctx %p is still alive\n", ctx);
924 	}
925 
926 	idr_destroy(&mgr->ctx_handles);
927 	mutex_destroy(&mgr->lock);
928 }
929 
930 void amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr *mgr,
931 			  ktime_t usage[AMDGPU_HW_IP_NUM])
932 {
933 	struct amdgpu_ctx *ctx;
934 	unsigned int hw_ip, i;
935 	uint32_t id;
936 
937 	/*
938 	 * This is a little bit racy because it can be that a ctx or a fence are
939 	 * destroyed just in the moment we try to account them. But that is ok
940 	 * since exactly that case is explicitely allowed by the interface.
941 	 */
942 	mutex_lock(&mgr->lock);
943 	for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
944 		uint64_t ns = atomic64_read(&mgr->time_spend[hw_ip]);
945 
946 		usage[hw_ip] = ns_to_ktime(ns);
947 	}
948 
949 	idr_for_each_entry(&mgr->ctx_handles, ctx, id) {
950 		for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
951 			for (i = 0; i < amdgpu_ctx_num_entities[hw_ip]; ++i) {
952 				struct amdgpu_ctx_entity *centity;
953 				ktime_t spend;
954 
955 				centity = ctx->entities[hw_ip][i];
956 				if (!centity)
957 					continue;
958 				spend = amdgpu_ctx_entity_time(ctx, centity);
959 				usage[hw_ip] = ktime_add(usage[hw_ip], spend);
960 			}
961 		}
962 	}
963 	mutex_unlock(&mgr->lock);
964 }
965