1d38ceaf9SAlex Deucher /*
2d38ceaf9SAlex Deucher * Copyright 2015 Advanced Micro Devices, Inc.
3d38ceaf9SAlex Deucher *
4d38ceaf9SAlex Deucher * Permission is hereby granted, free of charge, to any person obtaining a
5d38ceaf9SAlex Deucher * copy of this software and associated documentation files (the "Software"),
6d38ceaf9SAlex Deucher * to deal in the Software without restriction, including without limitation
7d38ceaf9SAlex Deucher * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8d38ceaf9SAlex Deucher * and/or sell copies of the Software, and to permit persons to whom the
9d38ceaf9SAlex Deucher * Software is furnished to do so, subject to the following conditions:
10d38ceaf9SAlex Deucher *
11d38ceaf9SAlex Deucher * The above copyright notice and this permission notice shall be included in
12d38ceaf9SAlex Deucher * all copies or substantial portions of the Software.
13d38ceaf9SAlex Deucher *
14d38ceaf9SAlex Deucher * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15d38ceaf9SAlex Deucher * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16d38ceaf9SAlex Deucher * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17d38ceaf9SAlex Deucher * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18d38ceaf9SAlex Deucher * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19d38ceaf9SAlex Deucher * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20d38ceaf9SAlex Deucher * OTHER DEALINGS IN THE SOFTWARE.
21d38ceaf9SAlex Deucher *
22d38ceaf9SAlex Deucher * Authors: monk liu <monk.liu@amd.com>
23d38ceaf9SAlex Deucher */
24d38ceaf9SAlex Deucher
25c2636dc5SAndres Rodriguez #include <drm/drm_auth.h>
2657230f0cSAndrey Grodzovsky #include <drm/drm_drv.h>
27d38ceaf9SAlex Deucher #include "amdgpu.h"
2852c6a62cSAndres Rodriguez #include "amdgpu_sched.h"
29ae363a21Sxinhui pan #include "amdgpu_ras.h"
301c6d567bSNirmoy Das #include <linux/nospec.h>
31d38ceaf9SAlex Deucher
321b1f2fecSChristian König #define to_amdgpu_ctx_entity(e) \
331b1f2fecSChristian König container_of((e), struct amdgpu_ctx_entity, entity)
341b1f2fecSChristian König
351b1f2fecSChristian König const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM] = {
361b1f2fecSChristian König [AMDGPU_HW_IP_GFX] = 1,
371b1f2fecSChristian König [AMDGPU_HW_IP_COMPUTE] = 4,
381b1f2fecSChristian König [AMDGPU_HW_IP_DMA] = 2,
391b1f2fecSChristian König [AMDGPU_HW_IP_UVD] = 1,
401b1f2fecSChristian König [AMDGPU_HW_IP_VCE] = 1,
411b1f2fecSChristian König [AMDGPU_HW_IP_UVD_ENC] = 1,
421b1f2fecSChristian König [AMDGPU_HW_IP_VCN_DEC] = 1,
431b1f2fecSChristian König [AMDGPU_HW_IP_VCN_ENC] = 1,
44f52c9643SAlex Deucher [AMDGPU_HW_IP_VCN_JPEG] = 1,
451b1f2fecSChristian König };
461b1f2fecSChristian König
amdgpu_ctx_priority_is_valid(int32_t ctx_prio)4784d588c3SNirmoy Das bool amdgpu_ctx_priority_is_valid(int32_t ctx_prio)
48c2636dc5SAndres Rodriguez {
4984d588c3SNirmoy Das switch (ctx_prio) {
5084d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_VERY_LOW:
5184d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_LOW:
5284d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_NORMAL:
5384d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_HIGH:
5484d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_VERY_HIGH:
5584d588c3SNirmoy Das return true;
5684d588c3SNirmoy Das default:
57eab02619SLuben Tuikov case AMDGPU_CTX_PRIORITY_UNSET:
58*d3df66fdSLuben Tuikov /* UNSET priority is not valid and we don't carry that
59*d3df66fdSLuben Tuikov * around, but set it to NORMAL in the only place this
60*d3df66fdSLuben Tuikov * function is called, amdgpu_ctx_ioctl().
61*d3df66fdSLuben Tuikov */
6284d588c3SNirmoy Das return false;
6384d588c3SNirmoy Das }
6484d588c3SNirmoy Das }
6584d588c3SNirmoy Das
6684d588c3SNirmoy Das static enum drm_sched_priority
amdgpu_ctx_to_drm_sched_prio(int32_t ctx_prio)6784d588c3SNirmoy Das amdgpu_ctx_to_drm_sched_prio(int32_t ctx_prio)
6884d588c3SNirmoy Das {
6984d588c3SNirmoy Das switch (ctx_prio) {
7084d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_UNSET:
71fa8391adSLuben Tuikov pr_warn_once("AMD-->DRM context priority value UNSET-->NORMAL");
72fa8391adSLuben Tuikov return DRM_SCHED_PRIORITY_NORMAL;
7384d588c3SNirmoy Das
7484d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_VERY_LOW:
7584d588c3SNirmoy Das return DRM_SCHED_PRIORITY_MIN;
7684d588c3SNirmoy Das
7784d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_LOW:
7884d588c3SNirmoy Das return DRM_SCHED_PRIORITY_MIN;
7984d588c3SNirmoy Das
8084d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_NORMAL:
8184d588c3SNirmoy Das return DRM_SCHED_PRIORITY_NORMAL;
8284d588c3SNirmoy Das
8384d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_HIGH:
8484d588c3SNirmoy Das return DRM_SCHED_PRIORITY_HIGH;
8584d588c3SNirmoy Das
8684d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_VERY_HIGH:
8784d588c3SNirmoy Das return DRM_SCHED_PRIORITY_HIGH;
8884d588c3SNirmoy Das
8984d588c3SNirmoy Das /* This should not happen as we sanitized userspace provided priority
9084d588c3SNirmoy Das * already, WARN if this happens.
9184d588c3SNirmoy Das */
9284d588c3SNirmoy Das default:
9384d588c3SNirmoy Das WARN(1, "Invalid context priority %d\n", ctx_prio);
9484d588c3SNirmoy Das return DRM_SCHED_PRIORITY_NORMAL;
9584d588c3SNirmoy Das }
9684d588c3SNirmoy Das
9784d588c3SNirmoy Das }
9884d588c3SNirmoy Das
amdgpu_ctx_priority_permit(struct drm_file * filp,int32_t priority)9984d588c3SNirmoy Das static int amdgpu_ctx_priority_permit(struct drm_file *filp,
10084d588c3SNirmoy Das int32_t priority)
10184d588c3SNirmoy Das {
102c2636dc5SAndres Rodriguez /* NORMAL and below are accessible by everyone */
10384d588c3SNirmoy Das if (priority <= AMDGPU_CTX_PRIORITY_NORMAL)
104c2636dc5SAndres Rodriguez return 0;
105c2636dc5SAndres Rodriguez
106c2636dc5SAndres Rodriguez if (capable(CAP_SYS_NICE))
107c2636dc5SAndres Rodriguez return 0;
108c2636dc5SAndres Rodriguez
109c2636dc5SAndres Rodriguez if (drm_is_current_master(filp))
110c2636dc5SAndres Rodriguez return 0;
111c2636dc5SAndres Rodriguez
112c2636dc5SAndres Rodriguez return -EACCES;
113c2636dc5SAndres Rodriguez }
114c2636dc5SAndres Rodriguez
amdgpu_ctx_prio_to_gfx_pipe_prio(int32_t prio)115b07d1d73SArunpravin Paneer Selvam static enum amdgpu_gfx_pipe_priority amdgpu_ctx_prio_to_gfx_pipe_prio(int32_t prio)
11633abcb1fSNirmoy Das {
11733abcb1fSNirmoy Das switch (prio) {
11884d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_HIGH:
11984d588c3SNirmoy Das case AMDGPU_CTX_PRIORITY_VERY_HIGH:
12033abcb1fSNirmoy Das return AMDGPU_GFX_PIPE_PRIO_HIGH;
12133abcb1fSNirmoy Das default:
12233abcb1fSNirmoy Das return AMDGPU_GFX_PIPE_PRIO_NORMAL;
12333abcb1fSNirmoy Das }
12433abcb1fSNirmoy Das }
12533abcb1fSNirmoy Das
amdgpu_ctx_sched_prio_to_ring_prio(int32_t prio)1267d7630fcSSatyajit Sahu static enum amdgpu_ring_priority_level amdgpu_ctx_sched_prio_to_ring_prio(int32_t prio)
1277d7630fcSSatyajit Sahu {
1287d7630fcSSatyajit Sahu switch (prio) {
1297d7630fcSSatyajit Sahu case AMDGPU_CTX_PRIORITY_HIGH:
1307d7630fcSSatyajit Sahu return AMDGPU_RING_PRIO_1;
1317d7630fcSSatyajit Sahu case AMDGPU_CTX_PRIORITY_VERY_HIGH:
1327d7630fcSSatyajit Sahu return AMDGPU_RING_PRIO_2;
1337d7630fcSSatyajit Sahu default:
1347d7630fcSSatyajit Sahu return AMDGPU_RING_PRIO_0;
1357d7630fcSSatyajit Sahu }
1367d7630fcSSatyajit Sahu }
1377d7630fcSSatyajit Sahu
amdgpu_ctx_get_hw_prio(struct amdgpu_ctx * ctx,u32 hw_ip)13884d588c3SNirmoy Das static unsigned int amdgpu_ctx_get_hw_prio(struct amdgpu_ctx *ctx, u32 hw_ip)
1391c6d567bSNirmoy Das {
14069493c03SChristian König struct amdgpu_device *adev = ctx->mgr->adev;
1411c6d567bSNirmoy Das unsigned int hw_prio;
14269493c03SChristian König int32_t ctx_prio;
1431c6d567bSNirmoy Das
14484d588c3SNirmoy Das ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
14584d588c3SNirmoy Das ctx->init_priority : ctx->override_priority;
14684d588c3SNirmoy Das
14784d588c3SNirmoy Das switch (hw_ip) {
148b07d1d73SArunpravin Paneer Selvam case AMDGPU_HW_IP_GFX:
14984d588c3SNirmoy Das case AMDGPU_HW_IP_COMPUTE:
150b07d1d73SArunpravin Paneer Selvam hw_prio = amdgpu_ctx_prio_to_gfx_pipe_prio(ctx_prio);
15184d588c3SNirmoy Das break;
1527d7630fcSSatyajit Sahu case AMDGPU_HW_IP_VCE:
1537d7630fcSSatyajit Sahu case AMDGPU_HW_IP_VCN_ENC:
1547d7630fcSSatyajit Sahu hw_prio = amdgpu_ctx_sched_prio_to_ring_prio(ctx_prio);
1557d7630fcSSatyajit Sahu break;
15684d588c3SNirmoy Das default:
15784d588c3SNirmoy Das hw_prio = AMDGPU_RING_PRIO_DEFAULT;
15884d588c3SNirmoy Das break;
15984d588c3SNirmoy Das }
16084d588c3SNirmoy Das
1611c6d567bSNirmoy Das hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
1621c6d567bSNirmoy Das if (adev->gpu_sched[hw_ip][hw_prio].num_scheds == 0)
1631c6d567bSNirmoy Das hw_prio = AMDGPU_RING_PRIO_DEFAULT;
1641c6d567bSNirmoy Das
1651c6d567bSNirmoy Das return hw_prio;
1661c6d567bSNirmoy Das }
1671c6d567bSNirmoy Das
168af0b5416SChristian König /* Calculate the time spend on the hw */
amdgpu_ctx_fence_time(struct dma_fence * fence)169af0b5416SChristian König static ktime_t amdgpu_ctx_fence_time(struct dma_fence *fence)
170af0b5416SChristian König {
171af0b5416SChristian König struct drm_sched_fence *s_fence;
172af0b5416SChristian König
173af0b5416SChristian König if (!fence)
174af0b5416SChristian König return ns_to_ktime(0);
175af0b5416SChristian König
176af0b5416SChristian König /* When the fence is not even scheduled it can't have spend time */
177af0b5416SChristian König s_fence = to_drm_sched_fence(fence);
178af0b5416SChristian König if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &s_fence->scheduled.flags))
179af0b5416SChristian König return ns_to_ktime(0);
180af0b5416SChristian König
181af0b5416SChristian König /* When it is still running account how much already spend */
182af0b5416SChristian König if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &s_fence->finished.flags))
183af0b5416SChristian König return ktime_sub(ktime_get(), s_fence->scheduled.timestamp);
184af0b5416SChristian König
185af0b5416SChristian König return ktime_sub(s_fence->finished.timestamp,
186af0b5416SChristian König s_fence->scheduled.timestamp);
187af0b5416SChristian König }
188af0b5416SChristian König
amdgpu_ctx_entity_time(struct amdgpu_ctx * ctx,struct amdgpu_ctx_entity * centity)189af0b5416SChristian König static ktime_t amdgpu_ctx_entity_time(struct amdgpu_ctx *ctx,
190af0b5416SChristian König struct amdgpu_ctx_entity *centity)
191af0b5416SChristian König {
192af0b5416SChristian König ktime_t res = ns_to_ktime(0);
193af0b5416SChristian König uint32_t i;
194af0b5416SChristian König
195af0b5416SChristian König spin_lock(&ctx->ring_lock);
196af0b5416SChristian König for (i = 0; i < amdgpu_sched_jobs; i++) {
197af0b5416SChristian König res = ktime_add(res, amdgpu_ctx_fence_time(centity->fences[i]));
198af0b5416SChristian König }
199af0b5416SChristian König spin_unlock(&ctx->ring_lock);
200af0b5416SChristian König return res;
201af0b5416SChristian König }
20284d588c3SNirmoy Das
amdgpu_ctx_init_entity(struct amdgpu_ctx * ctx,u32 hw_ip,const u32 ring)2031c6d567bSNirmoy Das static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip,
2041c6d567bSNirmoy Das const u32 ring)
205d38ceaf9SAlex Deucher {
206af0b5416SChristian König struct drm_gpu_scheduler **scheds = NULL, *sched = NULL;
20769493c03SChristian König struct amdgpu_device *adev = ctx->mgr->adev;
208977f7e10SNirmoy Das struct amdgpu_ctx_entity *entity;
20984d588c3SNirmoy Das enum drm_sched_priority drm_prio;
210af0b5416SChristian König unsigned int hw_prio, num_scheds;
211af0b5416SChristian König int32_t ctx_prio;
21247f38501SChristian König int r;
21347f38501SChristian König
214201a4eb9SGustavo A. R. Silva entity = kzalloc(struct_size(entity, fences, amdgpu_sched_jobs),
2151b1f2fecSChristian König GFP_KERNEL);
216977f7e10SNirmoy Das if (!entity)
21763e3ab9aSNirmoy Das return -ENOMEM;
21863e3ab9aSNirmoy Das
21984d588c3SNirmoy Das ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
220977f7e10SNirmoy Das ctx->init_priority : ctx->override_priority;
221af0b5416SChristian König entity->hw_ip = hw_ip;
22284d588c3SNirmoy Das entity->sequence = 1;
22384d588c3SNirmoy Das hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip);
22484d588c3SNirmoy Das drm_prio = amdgpu_ctx_to_drm_sched_prio(ctx_prio);
2251c6d567bSNirmoy Das
2261c6d567bSNirmoy Das hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM);
2279a18292dSJames Zhu
2289a18292dSJames Zhu if (!(adev)->xcp_mgr) {
2291c6d567bSNirmoy Das scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
2301c6d567bSNirmoy Das num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
2319a18292dSJames Zhu } else {
2329a18292dSJames Zhu struct amdgpu_fpriv *fpriv;
2339a18292dSJames Zhu
2349a18292dSJames Zhu fpriv = container_of(ctx->ctx_mgr, struct amdgpu_fpriv, ctx_mgr);
2359a18292dSJames Zhu r = amdgpu_xcp_select_scheds(adev, hw_ip, hw_prio, fpriv,
2369a18292dSJames Zhu &num_scheds, &scheds);
2379a18292dSJames Zhu if (r)
2389a18292dSJames Zhu goto cleanup_entity;
2399a18292dSJames Zhu }
2401c6d567bSNirmoy Das
241bc21585fSNirmoy Das /* disable load balance if the hw engine retains context among dependent jobs */
242bc21585fSNirmoy Das if (hw_ip == AMDGPU_HW_IP_VCN_ENC ||
243bc21585fSNirmoy Das hw_ip == AMDGPU_HW_IP_VCN_DEC ||
244bc21585fSNirmoy Das hw_ip == AMDGPU_HW_IP_UVD_ENC ||
245bc21585fSNirmoy Das hw_ip == AMDGPU_HW_IP_UVD) {
2461c6d567bSNirmoy Das sched = drm_sched_pick_best(scheds, num_scheds);
2470a96afc7SLe Ma scheds = &sched;
248f880799dSNirmoy Das num_scheds = 1;
249845e6fdfSChristian König }
250845e6fdfSChristian König
25184d588c3SNirmoy Das r = drm_sched_entity_init(&entity->entity, drm_prio, scheds, num_scheds,
252977f7e10SNirmoy Das &ctx->guilty);
2539cb7e5a9SChunming Zhou if (r)
254977f7e10SNirmoy Das goto error_free_entity;
255977f7e10SNirmoy Das
256d18b8eadSChristian König /* It's not an error if we fail to install the new entity */
257d18b8eadSChristian König if (cmpxchg(&ctx->entities[hw_ip][ring], NULL, entity))
258d18b8eadSChristian König goto cleanup_entity;
259d18b8eadSChristian König
260977f7e10SNirmoy Das return 0;
261977f7e10SNirmoy Das
262d18b8eadSChristian König cleanup_entity:
263d18b8eadSChristian König drm_sched_entity_fini(&entity->entity);
264d18b8eadSChristian König
265977f7e10SNirmoy Das error_free_entity:
266977f7e10SNirmoy Das kfree(entity);
267977f7e10SNirmoy Das
268977f7e10SNirmoy Das return r;
2699cb7e5a9SChunming Zhou }
2709cb7e5a9SChunming Zhou
amdgpu_ctx_fini_entity(struct amdgpu_device * adev,struct amdgpu_ctx_entity * entity)2713e7c6fe3SJames Zhu static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_device *adev,
2723e7c6fe3SJames Zhu struct amdgpu_ctx_entity *entity)
273af0b5416SChristian König {
274af0b5416SChristian König ktime_t res = ns_to_ktime(0);
275af0b5416SChristian König int i;
276af0b5416SChristian König
277af0b5416SChristian König if (!entity)
278af0b5416SChristian König return res;
279af0b5416SChristian König
280af0b5416SChristian König for (i = 0; i < amdgpu_sched_jobs; ++i) {
281af0b5416SChristian König res = ktime_add(res, amdgpu_ctx_fence_time(entity->fences[i]));
282af0b5416SChristian König dma_fence_put(entity->fences[i]);
283af0b5416SChristian König }
284af0b5416SChristian König
2853e7c6fe3SJames Zhu amdgpu_xcp_release_sched(adev, entity);
2863e7c6fe3SJames Zhu
287af0b5416SChristian König kfree(entity);
288af0b5416SChristian König return res;
289af0b5416SChristian König }
290af0b5416SChristian König
amdgpu_ctx_get_stable_pstate(struct amdgpu_ctx * ctx,u32 * stable_pstate)2918cda7a4fSAlex Deucher static int amdgpu_ctx_get_stable_pstate(struct amdgpu_ctx *ctx,
2928cda7a4fSAlex Deucher u32 *stable_pstate)
2938cda7a4fSAlex Deucher {
29469493c03SChristian König struct amdgpu_device *adev = ctx->mgr->adev;
2958cda7a4fSAlex Deucher enum amd_dpm_forced_level current_level;
2968cda7a4fSAlex Deucher
2978cda7a4fSAlex Deucher current_level = amdgpu_dpm_get_performance_level(adev);
2988cda7a4fSAlex Deucher
2998cda7a4fSAlex Deucher switch (current_level) {
3008cda7a4fSAlex Deucher case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
3018cda7a4fSAlex Deucher *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_STANDARD;
3028cda7a4fSAlex Deucher break;
3038cda7a4fSAlex Deucher case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
3048cda7a4fSAlex Deucher *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK;
3058cda7a4fSAlex Deucher break;
3068cda7a4fSAlex Deucher case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
3078cda7a4fSAlex Deucher *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK;
3088cda7a4fSAlex Deucher break;
3098cda7a4fSAlex Deucher case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
3108cda7a4fSAlex Deucher *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_PEAK;
3118cda7a4fSAlex Deucher break;
3128cda7a4fSAlex Deucher default:
3138cda7a4fSAlex Deucher *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE;
3148cda7a4fSAlex Deucher break;
3158cda7a4fSAlex Deucher }
3168cda7a4fSAlex Deucher return 0;
3178cda7a4fSAlex Deucher }
3188cda7a4fSAlex Deucher
amdgpu_ctx_init(struct amdgpu_ctx_mgr * mgr,int32_t priority,struct drm_file * filp,struct amdgpu_ctx * ctx)319958afce9SAlex Deucher static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority,
320958afce9SAlex Deucher struct drm_file *filp, struct amdgpu_ctx *ctx)
321958afce9SAlex Deucher {
322c30e326eSJames Zhu struct amdgpu_fpriv *fpriv = filp->driver_priv;
323958afce9SAlex Deucher u32 current_stable_pstate;
324958afce9SAlex Deucher int r;
325958afce9SAlex Deucher
326958afce9SAlex Deucher r = amdgpu_ctx_priority_permit(filp, priority);
327958afce9SAlex Deucher if (r)
328958afce9SAlex Deucher return r;
329958afce9SAlex Deucher
330958afce9SAlex Deucher memset(ctx, 0, sizeof(*ctx));
331958afce9SAlex Deucher
332958afce9SAlex Deucher kref_init(&ctx->refcount);
333958afce9SAlex Deucher ctx->mgr = mgr;
334958afce9SAlex Deucher spin_lock_init(&ctx->ring_lock);
335958afce9SAlex Deucher
336958afce9SAlex Deucher ctx->reset_counter = atomic_read(&mgr->adev->gpu_reset_counter);
337958afce9SAlex Deucher ctx->reset_counter_query = ctx->reset_counter;
338f88e295eSChristian König ctx->generation = amdgpu_vm_generation(mgr->adev, &fpriv->vm);
339958afce9SAlex Deucher ctx->init_priority = priority;
340958afce9SAlex Deucher ctx->override_priority = AMDGPU_CTX_PRIORITY_UNSET;
341958afce9SAlex Deucher
342958afce9SAlex Deucher r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate);
343958afce9SAlex Deucher if (r)
344958afce9SAlex Deucher return r;
345958afce9SAlex Deucher
34679610d30SChengming Gui if (mgr->adev->pm.stable_pstate_ctx)
34779610d30SChengming Gui ctx->stable_pstate = mgr->adev->pm.stable_pstate_ctx->stable_pstate;
34879610d30SChengming Gui else
349958afce9SAlex Deucher ctx->stable_pstate = current_stable_pstate;
350958afce9SAlex Deucher
351c30e326eSJames Zhu ctx->ctx_mgr = &(fpriv->ctx_mgr);
352958afce9SAlex Deucher return 0;
353958afce9SAlex Deucher }
354958afce9SAlex Deucher
amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx * ctx,u32 stable_pstate)3558cda7a4fSAlex Deucher static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx,
3568cda7a4fSAlex Deucher u32 stable_pstate)
3578cda7a4fSAlex Deucher {
35869493c03SChristian König struct amdgpu_device *adev = ctx->mgr->adev;
3598cda7a4fSAlex Deucher enum amd_dpm_forced_level level;
360505c170bSAlex Deucher u32 current_stable_pstate;
3618cda7a4fSAlex Deucher int r;
3628cda7a4fSAlex Deucher
3638cda7a4fSAlex Deucher mutex_lock(&adev->pm.stable_pstate_ctx_lock);
3648cda7a4fSAlex Deucher if (adev->pm.stable_pstate_ctx && adev->pm.stable_pstate_ctx != ctx) {
3658cda7a4fSAlex Deucher r = -EBUSY;
3668cda7a4fSAlex Deucher goto done;
3678cda7a4fSAlex Deucher }
3688cda7a4fSAlex Deucher
369505c170bSAlex Deucher r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate);
370505c170bSAlex Deucher if (r || (stable_pstate == current_stable_pstate))
371505c170bSAlex Deucher goto done;
372505c170bSAlex Deucher
3738cda7a4fSAlex Deucher switch (stable_pstate) {
3748cda7a4fSAlex Deucher case AMDGPU_CTX_STABLE_PSTATE_NONE:
3758cda7a4fSAlex Deucher level = AMD_DPM_FORCED_LEVEL_AUTO;
3768cda7a4fSAlex Deucher break;
3778cda7a4fSAlex Deucher case AMDGPU_CTX_STABLE_PSTATE_STANDARD:
3788cda7a4fSAlex Deucher level = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD;
3798cda7a4fSAlex Deucher break;
3808cda7a4fSAlex Deucher case AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK:
3818cda7a4fSAlex Deucher level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK;
3828cda7a4fSAlex Deucher break;
3838cda7a4fSAlex Deucher case AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK:
3848cda7a4fSAlex Deucher level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK;
3858cda7a4fSAlex Deucher break;
3868cda7a4fSAlex Deucher case AMDGPU_CTX_STABLE_PSTATE_PEAK:
3878cda7a4fSAlex Deucher level = AMD_DPM_FORCED_LEVEL_PROFILE_PEAK;
3888cda7a4fSAlex Deucher break;
3898cda7a4fSAlex Deucher default:
3908cda7a4fSAlex Deucher r = -EINVAL;
3918cda7a4fSAlex Deucher goto done;
3928cda7a4fSAlex Deucher }
3938cda7a4fSAlex Deucher
3948cda7a4fSAlex Deucher r = amdgpu_dpm_force_performance_level(adev, level);
3958cda7a4fSAlex Deucher
3968cda7a4fSAlex Deucher if (level == AMD_DPM_FORCED_LEVEL_AUTO)
3978cda7a4fSAlex Deucher adev->pm.stable_pstate_ctx = NULL;
3988cda7a4fSAlex Deucher else
3998cda7a4fSAlex Deucher adev->pm.stable_pstate_ctx = ctx;
4008cda7a4fSAlex Deucher done:
4018cda7a4fSAlex Deucher mutex_unlock(&adev->pm.stable_pstate_ctx_lock);
4028cda7a4fSAlex Deucher
4038cda7a4fSAlex Deucher return r;
4048cda7a4fSAlex Deucher }
4058cda7a4fSAlex Deucher
amdgpu_ctx_fini(struct kref * ref)4068ee3a52eSEmily Deng static void amdgpu_ctx_fini(struct kref *ref)
40747f38501SChristian König {
4088ee3a52eSEmily Deng struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount);
40969493c03SChristian König struct amdgpu_ctx_mgr *mgr = ctx->mgr;
41069493c03SChristian König struct amdgpu_device *adev = mgr->adev;
41157230f0cSAndrey Grodzovsky unsigned i, j, idx;
41247f38501SChristian König
413fe295b27SDave Airlie if (!adev)
414fe295b27SDave Airlie return;
415fe295b27SDave Airlie
416977f7e10SNirmoy Das for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
417977f7e10SNirmoy Das for (j = 0; j < AMDGPU_MAX_ENTITY_NUM; ++j) {
418af0b5416SChristian König ktime_t spend;
419af0b5416SChristian König
4203e7c6fe3SJames Zhu spend = amdgpu_ctx_fini_entity(adev, ctx->entities[i][j]);
421af0b5416SChristian König atomic64_add(ktime_to_ns(spend), &mgr->time_spend[i]);
422977f7e10SNirmoy Das }
42363e3ab9aSNirmoy Das }
42457230f0cSAndrey Grodzovsky
425a79f56d1SGuchun Chen if (drm_dev_enter(adev_to_drm(adev), &idx)) {
426958afce9SAlex Deucher amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate);
42757230f0cSAndrey Grodzovsky drm_dev_exit(idx);
42857230f0cSAndrey Grodzovsky }
42957230f0cSAndrey Grodzovsky
4308ee3a52eSEmily Deng kfree(ctx);
43147f38501SChristian König }
43247f38501SChristian König
amdgpu_ctx_get_entity(struct amdgpu_ctx * ctx,u32 hw_ip,u32 instance,u32 ring,struct drm_sched_entity ** entity)4330d346a14SChristian König int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance,
4340d346a14SChristian König u32 ring, struct drm_sched_entity **entity)
435869a53d4SChristian König {
436977f7e10SNirmoy Das int r;
43771eaac36SZhenGuo Yin struct drm_sched_entity *ctx_entity;
438977f7e10SNirmoy Das
4391b1f2fecSChristian König if (hw_ip >= AMDGPU_HW_IP_NUM) {
4401b1f2fecSChristian König DRM_ERROR("unknown HW IP type: %d\n", hw_ip);
4411b1f2fecSChristian König return -EINVAL;
4421b1f2fecSChristian König }
443869a53d4SChristian König
444869a53d4SChristian König /* Right now all IPs have only one instance - multiple rings. */
445869a53d4SChristian König if (instance != 0) {
446869a53d4SChristian König DRM_DEBUG("invalid ip instance: %d\n", instance);
447869a53d4SChristian König return -EINVAL;
448869a53d4SChristian König }
449869a53d4SChristian König
4501b1f2fecSChristian König if (ring >= amdgpu_ctx_num_entities[hw_ip]) {
4511b1f2fecSChristian König DRM_DEBUG("invalid ring: %d %d\n", hw_ip, ring);
452869a53d4SChristian König return -EINVAL;
453869a53d4SChristian König }
454869a53d4SChristian König
455977f7e10SNirmoy Das if (ctx->entities[hw_ip][ring] == NULL) {
456977f7e10SNirmoy Das r = amdgpu_ctx_init_entity(ctx, hw_ip, ring);
457977f7e10SNirmoy Das if (r)
458977f7e10SNirmoy Das return r;
459977f7e10SNirmoy Das }
460977f7e10SNirmoy Das
46171eaac36SZhenGuo Yin ctx_entity = &ctx->entities[hw_ip][ring]->entity;
46271eaac36SZhenGuo Yin r = drm_sched_entity_error(ctx_entity);
46371eaac36SZhenGuo Yin if (r) {
46471eaac36SZhenGuo Yin DRM_DEBUG("error entity %p\n", ctx_entity);
46571eaac36SZhenGuo Yin return r;
46671eaac36SZhenGuo Yin }
46771eaac36SZhenGuo Yin
46871eaac36SZhenGuo Yin *entity = ctx_entity;
469869a53d4SChristian König return 0;
470869a53d4SChristian König }
471869a53d4SChristian König
amdgpu_ctx_alloc(struct amdgpu_device * adev,struct amdgpu_fpriv * fpriv,struct drm_file * filp,int32_t priority,uint32_t * id)47247f38501SChristian König static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
47347f38501SChristian König struct amdgpu_fpriv *fpriv,
474c2636dc5SAndres Rodriguez struct drm_file *filp,
47584d588c3SNirmoy Das int32_t priority,
47647f38501SChristian König uint32_t *id)
47747f38501SChristian König {
47847f38501SChristian König struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
47947f38501SChristian König struct amdgpu_ctx *ctx;
48047f38501SChristian König int r;
48147f38501SChristian König
48247f38501SChristian König ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
48347f38501SChristian König if (!ctx)
48447f38501SChristian König return -ENOMEM;
48547f38501SChristian König
48647f38501SChristian König mutex_lock(&mgr->lock);
48708d1bdd4SRex Zhu r = idr_alloc(&mgr->ctx_handles, ctx, 1, AMDGPU_VM_MAX_NUM_CTX, GFP_KERNEL);
48847f38501SChristian König if (r < 0) {
48947f38501SChristian König mutex_unlock(&mgr->lock);
49047f38501SChristian König kfree(ctx);
49147f38501SChristian König return r;
49247f38501SChristian König }
493c2636dc5SAndres Rodriguez
49447f38501SChristian König *id = (uint32_t)r;
49569493c03SChristian König r = amdgpu_ctx_init(mgr, priority, filp, ctx);
496c648ed7cSChunming Zhou if (r) {
497c648ed7cSChunming Zhou idr_remove(&mgr->ctx_handles, *id);
498c648ed7cSChunming Zhou *id = 0;
499c648ed7cSChunming Zhou kfree(ctx);
500c648ed7cSChunming Zhou }
50147f38501SChristian König mutex_unlock(&mgr->lock);
50247f38501SChristian König return r;
50347f38501SChristian König }
50447f38501SChristian König
amdgpu_ctx_do_release(struct kref * ref)50547f38501SChristian König static void amdgpu_ctx_do_release(struct kref *ref)
506d38ceaf9SAlex Deucher {
507d38ceaf9SAlex Deucher struct amdgpu_ctx *ctx;
508977f7e10SNirmoy Das u32 i, j;
509d38ceaf9SAlex Deucher
51047f38501SChristian König ctx = container_of(ref, struct amdgpu_ctx, refcount);
511977f7e10SNirmoy Das for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
512977f7e10SNirmoy Das for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
513977f7e10SNirmoy Das if (!ctx->entities[i][j])
514977f7e10SNirmoy Das continue;
51547f38501SChristian König
516977f7e10SNirmoy Das drm_sched_entity_destroy(&ctx->entities[i][j]->entity);
517977f7e10SNirmoy Das }
518977f7e10SNirmoy Das }
51947f38501SChristian König
5208ee3a52eSEmily Deng amdgpu_ctx_fini(ref);
52147f38501SChristian König }
52247f38501SChristian König
amdgpu_ctx_free(struct amdgpu_fpriv * fpriv,uint32_t id)52347f38501SChristian König static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id)
52447f38501SChristian König {
52523ca0e4eSChunming Zhou struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
52647f38501SChristian König struct amdgpu_ctx *ctx;
52747f38501SChristian König
5280147ee0fSMarek Olšák mutex_lock(&mgr->lock);
529d3e709e6SMatthew Wilcox ctx = idr_remove(&mgr->ctx_handles, id);
530d3e709e6SMatthew Wilcox if (ctx)
531f11358daSMarek Olšák kref_put(&ctx->refcount, amdgpu_ctx_do_release);
5320147ee0fSMarek Olšák mutex_unlock(&mgr->lock);
533d3e709e6SMatthew Wilcox return ctx ? 0 : -EINVAL;
534d38ceaf9SAlex Deucher }
535d38ceaf9SAlex Deucher
amdgpu_ctx_query(struct amdgpu_device * adev,struct amdgpu_fpriv * fpriv,uint32_t id,union drm_amdgpu_ctx_out * out)536d94aed5aSMarek Olšák static int amdgpu_ctx_query(struct amdgpu_device *adev,
537d94aed5aSMarek Olšák struct amdgpu_fpriv *fpriv, uint32_t id,
538d94aed5aSMarek Olšák union drm_amdgpu_ctx_out *out)
539d38ceaf9SAlex Deucher {
540d38ceaf9SAlex Deucher struct amdgpu_ctx *ctx;
54123ca0e4eSChunming Zhou struct amdgpu_ctx_mgr *mgr;
542d94aed5aSMarek Olšák unsigned reset_counter;
543d38ceaf9SAlex Deucher
54423ca0e4eSChunming Zhou if (!fpriv)
54523ca0e4eSChunming Zhou return -EINVAL;
54623ca0e4eSChunming Zhou
54723ca0e4eSChunming Zhou mgr = &fpriv->ctx_mgr;
5480147ee0fSMarek Olšák mutex_lock(&mgr->lock);
549d38ceaf9SAlex Deucher ctx = idr_find(&mgr->ctx_handles, id);
550d94aed5aSMarek Olšák if (!ctx) {
5510147ee0fSMarek Olšák mutex_unlock(&mgr->lock);
552d38ceaf9SAlex Deucher return -EINVAL;
553d38ceaf9SAlex Deucher }
554d38ceaf9SAlex Deucher
555d94aed5aSMarek Olšák /* TODO: these two are always zero */
5560b492a4cSAlex Deucher out->state.flags = 0x0;
5570b492a4cSAlex Deucher out->state.hangs = 0x0;
558d94aed5aSMarek Olšák
559d94aed5aSMarek Olšák /* determine if a GPU reset has occured since the last call */
560d94aed5aSMarek Olšák reset_counter = atomic_read(&adev->gpu_reset_counter);
561d94aed5aSMarek Olšák /* TODO: this should ideally return NO, GUILTY, or INNOCENT. */
562668ca1b4SMonk Liu if (ctx->reset_counter_query == reset_counter)
563d94aed5aSMarek Olšák out->state.reset_status = AMDGPU_CTX_NO_RESET;
564d94aed5aSMarek Olšák else
565d94aed5aSMarek Olšák out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET;
566668ca1b4SMonk Liu ctx->reset_counter_query = reset_counter;
567d94aed5aSMarek Olšák
568d94aed5aSMarek Olšák mutex_unlock(&mgr->lock);
569d94aed5aSMarek Olšák return 0;
570d94aed5aSMarek Olšák }
571d94aed5aSMarek Olšák
57205adfd80SLuben Tuikov #define AMDGPU_RAS_COUNTE_DELAY_MS 3000
57305adfd80SLuben Tuikov
amdgpu_ctx_query2(struct amdgpu_device * adev,struct amdgpu_fpriv * fpriv,uint32_t id,union drm_amdgpu_ctx_out * out)574bc1b1bf6SMonk Liu static int amdgpu_ctx_query2(struct amdgpu_device *adev,
575bc1b1bf6SMonk Liu struct amdgpu_fpriv *fpriv, uint32_t id,
576bc1b1bf6SMonk Liu union drm_amdgpu_ctx_out *out)
577bc1b1bf6SMonk Liu {
57805adfd80SLuben Tuikov struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
579bc1b1bf6SMonk Liu struct amdgpu_ctx *ctx;
580bc1b1bf6SMonk Liu struct amdgpu_ctx_mgr *mgr;
581bc1b1bf6SMonk Liu
582bc1b1bf6SMonk Liu if (!fpriv)
583bc1b1bf6SMonk Liu return -EINVAL;
584bc1b1bf6SMonk Liu
585bc1b1bf6SMonk Liu mgr = &fpriv->ctx_mgr;
586bc1b1bf6SMonk Liu mutex_lock(&mgr->lock);
587bc1b1bf6SMonk Liu ctx = idr_find(&mgr->ctx_handles, id);
588bc1b1bf6SMonk Liu if (!ctx) {
589bc1b1bf6SMonk Liu mutex_unlock(&mgr->lock);
590bc1b1bf6SMonk Liu return -EINVAL;
591bc1b1bf6SMonk Liu }
592bc1b1bf6SMonk Liu
593bc1b1bf6SMonk Liu out->state.flags = 0x0;
594bc1b1bf6SMonk Liu out->state.hangs = 0x0;
595bc1b1bf6SMonk Liu
596bc1b1bf6SMonk Liu if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter))
597bc1b1bf6SMonk Liu out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET;
598bc1b1bf6SMonk Liu
599f88e295eSChristian König if (ctx->generation != amdgpu_vm_generation(adev, &fpriv->vm))
600bc1b1bf6SMonk Liu out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST;
601bc1b1bf6SMonk Liu
602bc1b1bf6SMonk Liu if (atomic_read(&ctx->guilty))
603bc1b1bf6SMonk Liu out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY;
604bc1b1bf6SMonk Liu
605489763afSPierre-Eric Pelloux-Prayer if (amdgpu_in_reset(adev))
606489763afSPierre-Eric Pelloux-Prayer out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET_IN_PROGRESS;
607489763afSPierre-Eric Pelloux-Prayer
60805adfd80SLuben Tuikov if (adev->ras_enabled && con) {
60905adfd80SLuben Tuikov /* Return the cached values in O(1),
61005adfd80SLuben Tuikov * and schedule delayed work to cache
61105adfd80SLuben Tuikov * new vaues.
61205adfd80SLuben Tuikov */
61305adfd80SLuben Tuikov int ce_count, ue_count;
61405adfd80SLuben Tuikov
61505adfd80SLuben Tuikov ce_count = atomic_read(&con->ras_ce_count);
61605adfd80SLuben Tuikov ue_count = atomic_read(&con->ras_ue_count);
61705adfd80SLuben Tuikov
61805adfd80SLuben Tuikov if (ce_count != ctx->ras_counter_ce) {
61905adfd80SLuben Tuikov ctx->ras_counter_ce = ce_count;
62005adfd80SLuben Tuikov out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_CE;
62105adfd80SLuben Tuikov }
62205adfd80SLuben Tuikov
62305adfd80SLuben Tuikov if (ue_count != ctx->ras_counter_ue) {
62405adfd80SLuben Tuikov ctx->ras_counter_ue = ue_count;
62505adfd80SLuben Tuikov out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_UE;
62605adfd80SLuben Tuikov }
62705adfd80SLuben Tuikov
62805adfd80SLuben Tuikov schedule_delayed_work(&con->ras_counte_delay_work,
62905adfd80SLuben Tuikov msecs_to_jiffies(AMDGPU_RAS_COUNTE_DELAY_MS));
63005adfd80SLuben Tuikov }
63105adfd80SLuben Tuikov
632bc1b1bf6SMonk Liu mutex_unlock(&mgr->lock);
633bc1b1bf6SMonk Liu return 0;
634bc1b1bf6SMonk Liu }
635bc1b1bf6SMonk Liu
amdgpu_ctx_stable_pstate(struct amdgpu_device * adev,struct amdgpu_fpriv * fpriv,uint32_t id,bool set,u32 * stable_pstate)6368cda7a4fSAlex Deucher static int amdgpu_ctx_stable_pstate(struct amdgpu_device *adev,
6378cda7a4fSAlex Deucher struct amdgpu_fpriv *fpriv, uint32_t id,
6388cda7a4fSAlex Deucher bool set, u32 *stable_pstate)
6398cda7a4fSAlex Deucher {
6408cda7a4fSAlex Deucher struct amdgpu_ctx *ctx;
6418cda7a4fSAlex Deucher struct amdgpu_ctx_mgr *mgr;
6428cda7a4fSAlex Deucher int r;
6438cda7a4fSAlex Deucher
6448cda7a4fSAlex Deucher if (!fpriv)
6458cda7a4fSAlex Deucher return -EINVAL;
6468cda7a4fSAlex Deucher
6478cda7a4fSAlex Deucher mgr = &fpriv->ctx_mgr;
6488cda7a4fSAlex Deucher mutex_lock(&mgr->lock);
6498cda7a4fSAlex Deucher ctx = idr_find(&mgr->ctx_handles, id);
6508cda7a4fSAlex Deucher if (!ctx) {
6518cda7a4fSAlex Deucher mutex_unlock(&mgr->lock);
6528cda7a4fSAlex Deucher return -EINVAL;
6538cda7a4fSAlex Deucher }
6548cda7a4fSAlex Deucher
6558cda7a4fSAlex Deucher if (set)
6568cda7a4fSAlex Deucher r = amdgpu_ctx_set_stable_pstate(ctx, *stable_pstate);
6578cda7a4fSAlex Deucher else
6588cda7a4fSAlex Deucher r = amdgpu_ctx_get_stable_pstate(ctx, stable_pstate);
6598cda7a4fSAlex Deucher
6608cda7a4fSAlex Deucher mutex_unlock(&mgr->lock);
6618cda7a4fSAlex Deucher return r;
6628cda7a4fSAlex Deucher }
6638cda7a4fSAlex Deucher
amdgpu_ctx_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)664d38ceaf9SAlex Deucher int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
665d38ceaf9SAlex Deucher struct drm_file *filp)
666d38ceaf9SAlex Deucher {
667d38ceaf9SAlex Deucher int r;
6688cda7a4fSAlex Deucher uint32_t id, stable_pstate;
66984d588c3SNirmoy Das int32_t priority;
670d38ceaf9SAlex Deucher
671d38ceaf9SAlex Deucher union drm_amdgpu_ctx *args = data;
6721348969aSLuben Tuikov struct amdgpu_device *adev = drm_to_adev(dev);
673d38ceaf9SAlex Deucher struct amdgpu_fpriv *fpriv = filp->driver_priv;
674d38ceaf9SAlex Deucher
675d38ceaf9SAlex Deucher id = args->in.ctx_id;
67684d588c3SNirmoy Das priority = args->in.priority;
677c2636dc5SAndres Rodriguez
678*d3df66fdSLuben Tuikov /* For backwards compatibility, we need to accept ioctls with garbage
679*d3df66fdSLuben Tuikov * in the priority field. Garbage values in the priority field, result
680*d3df66fdSLuben Tuikov * in the priority being set to NORMAL.
681*d3df66fdSLuben Tuikov */
68284d588c3SNirmoy Das if (!amdgpu_ctx_priority_is_valid(priority))
68384d588c3SNirmoy Das priority = AMDGPU_CTX_PRIORITY_NORMAL;
684d38ceaf9SAlex Deucher
685d38ceaf9SAlex Deucher switch (args->in.op) {
686d38ceaf9SAlex Deucher case AMDGPU_CTX_OP_ALLOC_CTX:
687c2636dc5SAndres Rodriguez r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id);
688d38ceaf9SAlex Deucher args->out.alloc.ctx_id = id;
689d38ceaf9SAlex Deucher break;
690d38ceaf9SAlex Deucher case AMDGPU_CTX_OP_FREE_CTX:
69147f38501SChristian König r = amdgpu_ctx_free(fpriv, id);
692d38ceaf9SAlex Deucher break;
693d38ceaf9SAlex Deucher case AMDGPU_CTX_OP_QUERY_STATE:
694d94aed5aSMarek Olšák r = amdgpu_ctx_query(adev, fpriv, id, &args->out);
695d38ceaf9SAlex Deucher break;
696bc1b1bf6SMonk Liu case AMDGPU_CTX_OP_QUERY_STATE2:
697bc1b1bf6SMonk Liu r = amdgpu_ctx_query2(adev, fpriv, id, &args->out);
698bc1b1bf6SMonk Liu break;
6998cda7a4fSAlex Deucher case AMDGPU_CTX_OP_GET_STABLE_PSTATE:
7008cda7a4fSAlex Deucher if (args->in.flags)
7018cda7a4fSAlex Deucher return -EINVAL;
7028cda7a4fSAlex Deucher r = amdgpu_ctx_stable_pstate(adev, fpriv, id, false, &stable_pstate);
703eed1a5c7STom Rix if (!r)
7048cda7a4fSAlex Deucher args->out.pstate.flags = stable_pstate;
7058cda7a4fSAlex Deucher break;
7068cda7a4fSAlex Deucher case AMDGPU_CTX_OP_SET_STABLE_PSTATE:
7078cda7a4fSAlex Deucher if (args->in.flags & ~AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK)
7088cda7a4fSAlex Deucher return -EINVAL;
7098cda7a4fSAlex Deucher stable_pstate = args->in.flags & AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK;
7108cda7a4fSAlex Deucher if (stable_pstate > AMDGPU_CTX_STABLE_PSTATE_PEAK)
7118cda7a4fSAlex Deucher return -EINVAL;
7128cda7a4fSAlex Deucher r = amdgpu_ctx_stable_pstate(adev, fpriv, id, true, &stable_pstate);
7138cda7a4fSAlex Deucher break;
714d38ceaf9SAlex Deucher default:
715d38ceaf9SAlex Deucher return -EINVAL;
716d38ceaf9SAlex Deucher }
717d38ceaf9SAlex Deucher
718d38ceaf9SAlex Deucher return r;
719d38ceaf9SAlex Deucher }
72066b3cf2aSJammy Zhou
amdgpu_ctx_get(struct amdgpu_fpriv * fpriv,uint32_t id)72166b3cf2aSJammy Zhou struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id)
72266b3cf2aSJammy Zhou {
72366b3cf2aSJammy Zhou struct amdgpu_ctx *ctx;
72423ca0e4eSChunming Zhou struct amdgpu_ctx_mgr *mgr;
72523ca0e4eSChunming Zhou
72623ca0e4eSChunming Zhou if (!fpriv)
72723ca0e4eSChunming Zhou return NULL;
72823ca0e4eSChunming Zhou
72923ca0e4eSChunming Zhou mgr = &fpriv->ctx_mgr;
73066b3cf2aSJammy Zhou
73166b3cf2aSJammy Zhou mutex_lock(&mgr->lock);
73266b3cf2aSJammy Zhou ctx = idr_find(&mgr->ctx_handles, id);
73366b3cf2aSJammy Zhou if (ctx)
73466b3cf2aSJammy Zhou kref_get(&ctx->refcount);
73566b3cf2aSJammy Zhou mutex_unlock(&mgr->lock);
73666b3cf2aSJammy Zhou return ctx;
73766b3cf2aSJammy Zhou }
73866b3cf2aSJammy Zhou
amdgpu_ctx_put(struct amdgpu_ctx * ctx)73966b3cf2aSJammy Zhou int amdgpu_ctx_put(struct amdgpu_ctx *ctx)
74066b3cf2aSJammy Zhou {
74166b3cf2aSJammy Zhou if (ctx == NULL)
74266b3cf2aSJammy Zhou return -EINVAL;
74366b3cf2aSJammy Zhou
74466b3cf2aSJammy Zhou kref_put(&ctx->refcount, amdgpu_ctx_do_release);
74566b3cf2aSJammy Zhou return 0;
74666b3cf2aSJammy Zhou }
74721c16bf6SChristian König
amdgpu_ctx_add_fence(struct amdgpu_ctx * ctx,struct drm_sched_entity * entity,struct dma_fence * fence)74869493c03SChristian König uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx,
7490d346a14SChristian König struct drm_sched_entity *entity,
75069493c03SChristian König struct dma_fence *fence)
75121c16bf6SChristian König {
7521b1f2fecSChristian König struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
7531b1f2fecSChristian König uint64_t seq = centity->sequence;
754f54d1867SChris Wilson struct dma_fence *other = NULL;
7550d346a14SChristian König unsigned idx = 0;
75621c16bf6SChristian König
7575b011235SChunming Zhou idx = seq & (amdgpu_sched_jobs - 1);
7581b1f2fecSChristian König other = centity->fences[idx];
75969493c03SChristian König WARN_ON(other && !dma_fence_is_signaled(other));
76021c16bf6SChristian König
761f54d1867SChris Wilson dma_fence_get(fence);
76221c16bf6SChristian König
76321c16bf6SChristian König spin_lock(&ctx->ring_lock);
7641b1f2fecSChristian König centity->fences[idx] = fence;
7651b1f2fecSChristian König centity->sequence++;
76621c16bf6SChristian König spin_unlock(&ctx->ring_lock);
76721c16bf6SChristian König
768af0b5416SChristian König atomic64_add(ktime_to_ns(amdgpu_ctx_fence_time(other)),
769af0b5416SChristian König &ctx->mgr->time_spend[centity->hw_ip]);
770af0b5416SChristian König
771f54d1867SChris Wilson dma_fence_put(other);
77269493c03SChristian König return seq;
77321c16bf6SChristian König }
77421c16bf6SChristian König
amdgpu_ctx_get_fence(struct amdgpu_ctx * ctx,struct drm_sched_entity * entity,uint64_t seq)775f54d1867SChris Wilson struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
7760d346a14SChristian König struct drm_sched_entity *entity,
7770d346a14SChristian König uint64_t seq)
77821c16bf6SChristian König {
7791b1f2fecSChristian König struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
780f54d1867SChris Wilson struct dma_fence *fence;
78121c16bf6SChristian König
78221c16bf6SChristian König spin_lock(&ctx->ring_lock);
783b43a9a7eSChunming Zhou
784d7b1eeb2SMonk Liu if (seq == ~0ull)
7851b1f2fecSChristian König seq = centity->sequence - 1;
786d7b1eeb2SMonk Liu
7871b1f2fecSChristian König if (seq >= centity->sequence) {
78821c16bf6SChristian König spin_unlock(&ctx->ring_lock);
78921c16bf6SChristian König return ERR_PTR(-EINVAL);
79021c16bf6SChristian König }
79121c16bf6SChristian König
792b43a9a7eSChunming Zhou
7931b1f2fecSChristian König if (seq + amdgpu_sched_jobs < centity->sequence) {
79421c16bf6SChristian König spin_unlock(&ctx->ring_lock);
79521c16bf6SChristian König return NULL;
79621c16bf6SChristian König }
79721c16bf6SChristian König
7981b1f2fecSChristian König fence = dma_fence_get(centity->fences[seq & (amdgpu_sched_jobs - 1)]);
79921c16bf6SChristian König spin_unlock(&ctx->ring_lock);
80021c16bf6SChristian König
80121c16bf6SChristian König return fence;
80221c16bf6SChristian König }
803efd4ccb5SChristian König
amdgpu_ctx_set_entity_priority(struct amdgpu_ctx * ctx,struct amdgpu_ctx_entity * aentity,int hw_ip,int32_t priority)8042316a86bSNirmoy Das static void amdgpu_ctx_set_entity_priority(struct amdgpu_ctx *ctx,
8052316a86bSNirmoy Das struct amdgpu_ctx_entity *aentity,
8062316a86bSNirmoy Das int hw_ip,
80784d588c3SNirmoy Das int32_t priority)
8082316a86bSNirmoy Das {
80969493c03SChristian König struct amdgpu_device *adev = ctx->mgr->adev;
8101c6d567bSNirmoy Das unsigned int hw_prio;
8112316a86bSNirmoy Das struct drm_gpu_scheduler **scheds = NULL;
8122316a86bSNirmoy Das unsigned num_scheds;
8132316a86bSNirmoy Das
8142316a86bSNirmoy Das /* set sw priority */
81584d588c3SNirmoy Das drm_sched_entity_set_priority(&aentity->entity,
81684d588c3SNirmoy Das amdgpu_ctx_to_drm_sched_prio(priority));
8172316a86bSNirmoy Das
8182316a86bSNirmoy Das /* set hw priority */
819b07d1d73SArunpravin Paneer Selvam if (hw_ip == AMDGPU_HW_IP_COMPUTE || hw_ip == AMDGPU_HW_IP_GFX) {
82084d588c3SNirmoy Das hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip);
8211c6d567bSNirmoy Das hw_prio = array_index_nospec(hw_prio, AMDGPU_RING_PRIO_MAX);
8221c6d567bSNirmoy Das scheds = adev->gpu_sched[hw_ip][hw_prio].sched;
8231c6d567bSNirmoy Das num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds;
8242316a86bSNirmoy Das drm_sched_entity_modify_sched(&aentity->entity, scheds,
8252316a86bSNirmoy Das num_scheds);
8262316a86bSNirmoy Das }
8272316a86bSNirmoy Das }
8282316a86bSNirmoy Das
amdgpu_ctx_priority_override(struct amdgpu_ctx * ctx,int32_t priority)829c23be4aeSAndres Rodriguez void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
83084d588c3SNirmoy Das int32_t priority)
831c23be4aeSAndres Rodriguez {
83284d588c3SNirmoy Das int32_t ctx_prio;
833977f7e10SNirmoy Das unsigned i, j;
834c23be4aeSAndres Rodriguez
835c23be4aeSAndres Rodriguez ctx->override_priority = priority;
836c23be4aeSAndres Rodriguez
83784d588c3SNirmoy Das ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ?
838c23be4aeSAndres Rodriguez ctx->init_priority : ctx->override_priority;
839977f7e10SNirmoy Das for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
840977f7e10SNirmoy Das for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
841977f7e10SNirmoy Das if (!ctx->entities[i][j])
842977f7e10SNirmoy Das continue;
843c23be4aeSAndres Rodriguez
8442316a86bSNirmoy Das amdgpu_ctx_set_entity_priority(ctx, ctx->entities[i][j],
8452316a86bSNirmoy Das i, ctx_prio);
846c23be4aeSAndres Rodriguez }
847c23be4aeSAndres Rodriguez }
848977f7e10SNirmoy Das }
849c23be4aeSAndres Rodriguez
amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx * ctx,struct drm_sched_entity * entity)8500d346a14SChristian König int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
8510d346a14SChristian König struct drm_sched_entity *entity)
8520ae94444SAndrey Grodzovsky {
8531b1f2fecSChristian König struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
85475e1cafdSChristian König struct dma_fence *other;
85575e1cafdSChristian König unsigned idx;
85675e1cafdSChristian König long r;
8570ae94444SAndrey Grodzovsky
85875e1cafdSChristian König spin_lock(&ctx->ring_lock);
85975e1cafdSChristian König idx = centity->sequence & (amdgpu_sched_jobs - 1);
86075e1cafdSChristian König other = dma_fence_get(centity->fences[idx]);
86175e1cafdSChristian König spin_unlock(&ctx->ring_lock);
86275e1cafdSChristian König
86375e1cafdSChristian König if (!other)
86475e1cafdSChristian König return 0;
86575e1cafdSChristian König
866719a39a1SAndrey Grodzovsky r = dma_fence_wait(other, true);
86775e1cafdSChristian König if (r < 0 && r != -ERESTARTSYS)
8680ae94444SAndrey Grodzovsky DRM_ERROR("Error (%ld) waiting for fence!\n", r);
869719a39a1SAndrey Grodzovsky
87075e1cafdSChristian König dma_fence_put(other);
8710ae94444SAndrey Grodzovsky return r;
8720ae94444SAndrey Grodzovsky }
8730ae94444SAndrey Grodzovsky
amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr * mgr,struct amdgpu_device * adev)87469493c03SChristian König void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr,
87569493c03SChristian König struct amdgpu_device *adev)
876efd4ccb5SChristian König {
877af0b5416SChristian König unsigned int i;
878af0b5416SChristian König
87969493c03SChristian König mgr->adev = adev;
880efd4ccb5SChristian König mutex_init(&mgr->lock);
8812ddd1e6cSDanilo Krummrich idr_init_base(&mgr->ctx_handles, 1);
882af0b5416SChristian König
883af0b5416SChristian König for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
884af0b5416SChristian König atomic64_set(&mgr->time_spend[i], 0);
885efd4ccb5SChristian König }
886efd4ccb5SChristian König
amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr * mgr,long timeout)88756753e73SChristian König long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout)
8888ee3a52eSEmily Deng {
8898ee3a52eSEmily Deng struct amdgpu_ctx *ctx;
8908ee3a52eSEmily Deng struct idr *idp;
891977f7e10SNirmoy Das uint32_t id, i, j;
8928ee3a52eSEmily Deng
8938ee3a52eSEmily Deng idp = &mgr->ctx_handles;
8948ee3a52eSEmily Deng
89548ad368aSAndrey Grodzovsky mutex_lock(&mgr->lock);
8968ee3a52eSEmily Deng idr_for_each_entry(idp, ctx, id) {
897977f7e10SNirmoy Das for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
898977f7e10SNirmoy Das for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
8991b1f2fecSChristian König struct drm_sched_entity *entity;
90020b6b788SAndrey Grodzovsky
901977f7e10SNirmoy Das if (!ctx->entities[i][j])
902977f7e10SNirmoy Das continue;
903977f7e10SNirmoy Das
904977f7e10SNirmoy Das entity = &ctx->entities[i][j]->entity;
90556753e73SChristian König timeout = drm_sched_entity_flush(entity, timeout);
9068ee3a52eSEmily Deng }
9078ee3a52eSEmily Deng }
908977f7e10SNirmoy Das }
90948ad368aSAndrey Grodzovsky mutex_unlock(&mgr->lock);
91056753e73SChristian König return timeout;
91120b6b788SAndrey Grodzovsky }
9128ee3a52eSEmily Deng
amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr * mgr)913c49d8280SAndrey Grodzovsky void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr)
9148ee3a52eSEmily Deng {
9158ee3a52eSEmily Deng struct amdgpu_ctx *ctx;
9168ee3a52eSEmily Deng struct idr *idp;
917977f7e10SNirmoy Das uint32_t id, i, j;
9188ee3a52eSEmily Deng
9198ee3a52eSEmily Deng idp = &mgr->ctx_handles;
9208ee3a52eSEmily Deng
9218ee3a52eSEmily Deng idr_for_each_entry(idp, ctx, id) {
9221b1f2fecSChristian König if (kref_read(&ctx->refcount) != 1) {
9238ee3a52eSEmily Deng DRM_ERROR("ctx %p is still alive\n", ctx);
9241b1f2fecSChristian König continue;
9258ee3a52eSEmily Deng }
9261b1f2fecSChristian König
927977f7e10SNirmoy Das for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) {
928977f7e10SNirmoy Das for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) {
929977f7e10SNirmoy Das struct drm_sched_entity *entity;
930977f7e10SNirmoy Das
931977f7e10SNirmoy Das if (!ctx->entities[i][j])
932977f7e10SNirmoy Das continue;
933977f7e10SNirmoy Das
934977f7e10SNirmoy Das entity = &ctx->entities[i][j]->entity;
935977f7e10SNirmoy Das drm_sched_entity_fini(entity);
936977f7e10SNirmoy Das }
937977f7e10SNirmoy Das }
9388ee3a52eSEmily Deng }
93920b6b788SAndrey Grodzovsky }
9408ee3a52eSEmily Deng
amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr * mgr)941efd4ccb5SChristian König void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
942efd4ccb5SChristian König {
943efd4ccb5SChristian König struct amdgpu_ctx *ctx;
944efd4ccb5SChristian König struct idr *idp;
945efd4ccb5SChristian König uint32_t id;
946efd4ccb5SChristian König
947c49d8280SAndrey Grodzovsky amdgpu_ctx_mgr_entity_fini(mgr);
9488ee3a52eSEmily Deng
949efd4ccb5SChristian König idp = &mgr->ctx_handles;
950efd4ccb5SChristian König
951efd4ccb5SChristian König idr_for_each_entry(idp, ctx, id) {
9528ee3a52eSEmily Deng if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1)
953efd4ccb5SChristian König DRM_ERROR("ctx %p is still alive\n", ctx);
954efd4ccb5SChristian König }
955efd4ccb5SChristian König
956efd4ccb5SChristian König idr_destroy(&mgr->ctx_handles);
957efd4ccb5SChristian König mutex_destroy(&mgr->lock);
958efd4ccb5SChristian König }
95987444254SRoy Sun
amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr * mgr,ktime_t usage[AMDGPU_HW_IP_NUM])960af0b5416SChristian König void amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr *mgr,
961af0b5416SChristian König ktime_t usage[AMDGPU_HW_IP_NUM])
96287444254SRoy Sun {
96387444254SRoy Sun struct amdgpu_ctx *ctx;
964af0b5416SChristian König unsigned int hw_ip, i;
96587444254SRoy Sun uint32_t id;
96687444254SRoy Sun
967af0b5416SChristian König /*
968af0b5416SChristian König * This is a little bit racy because it can be that a ctx or a fence are
969af0b5416SChristian König * destroyed just in the moment we try to account them. But that is ok
970af0b5416SChristian König * since exactly that case is explicitely allowed by the interface.
9715c439c38SDavid M Nieto */
972af0b5416SChristian König mutex_lock(&mgr->lock);
973af0b5416SChristian König for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
974af0b5416SChristian König uint64_t ns = atomic64_read(&mgr->time_spend[hw_ip]);
9755c439c38SDavid M Nieto
976af0b5416SChristian König usage[hw_ip] = ns_to_ktime(ns);
97787444254SRoy Sun }
97887444254SRoy Sun
979af0b5416SChristian König idr_for_each_entry(&mgr->ctx_handles, ctx, id) {
980af0b5416SChristian König for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) {
981af0b5416SChristian König for (i = 0; i < amdgpu_ctx_num_entities[hw_ip]; ++i) {
982af0b5416SChristian König struct amdgpu_ctx_entity *centity;
983af0b5416SChristian König ktime_t spend;
98487444254SRoy Sun
985af0b5416SChristian König centity = ctx->entities[hw_ip][i];
986af0b5416SChristian König if (!centity)
987af0b5416SChristian König continue;
988af0b5416SChristian König spend = amdgpu_ctx_entity_time(ctx, centity);
989af0b5416SChristian König usage[hw_ip] = ktime_add(usage[hw_ip], spend);
990af0b5416SChristian König }
991af0b5416SChristian König }
992af0b5416SChristian König }
993af0b5416SChristian König mutex_unlock(&mgr->lock);
99487444254SRoy Sun }
995