1d38ceaf9SAlex Deucher /* 2d38ceaf9SAlex Deucher * Copyright 2015 Advanced Micro Devices, Inc. 3d38ceaf9SAlex Deucher * 4d38ceaf9SAlex Deucher * Permission is hereby granted, free of charge, to any person obtaining a 5d38ceaf9SAlex Deucher * copy of this software and associated documentation files (the "Software"), 6d38ceaf9SAlex Deucher * to deal in the Software without restriction, including without limitation 7d38ceaf9SAlex Deucher * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8d38ceaf9SAlex Deucher * and/or sell copies of the Software, and to permit persons to whom the 9d38ceaf9SAlex Deucher * Software is furnished to do so, subject to the following conditions: 10d38ceaf9SAlex Deucher * 11d38ceaf9SAlex Deucher * The above copyright notice and this permission notice shall be included in 12d38ceaf9SAlex Deucher * all copies or substantial portions of the Software. 13d38ceaf9SAlex Deucher * 14d38ceaf9SAlex Deucher * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15d38ceaf9SAlex Deucher * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16d38ceaf9SAlex Deucher * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17d38ceaf9SAlex Deucher * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18d38ceaf9SAlex Deucher * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19d38ceaf9SAlex Deucher * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20d38ceaf9SAlex Deucher * OTHER DEALINGS IN THE SOFTWARE. 21d38ceaf9SAlex Deucher * 22d38ceaf9SAlex Deucher * Authors: monk liu <monk.liu@amd.com> 23d38ceaf9SAlex Deucher */ 24d38ceaf9SAlex Deucher 25c2636dc5SAndres Rodriguez #include <drm/drm_auth.h> 26d38ceaf9SAlex Deucher #include "amdgpu.h" 2752c6a62cSAndres Rodriguez #include "amdgpu_sched.h" 28ae363a21Sxinhui pan #include "amdgpu_ras.h" 291c6d567bSNirmoy Das #include <linux/nospec.h> 30d38ceaf9SAlex Deucher 311b1f2fecSChristian König #define to_amdgpu_ctx_entity(e) \ 321b1f2fecSChristian König container_of((e), struct amdgpu_ctx_entity, entity) 331b1f2fecSChristian König 341b1f2fecSChristian König const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM] = { 351b1f2fecSChristian König [AMDGPU_HW_IP_GFX] = 1, 361b1f2fecSChristian König [AMDGPU_HW_IP_COMPUTE] = 4, 371b1f2fecSChristian König [AMDGPU_HW_IP_DMA] = 2, 381b1f2fecSChristian König [AMDGPU_HW_IP_UVD] = 1, 391b1f2fecSChristian König [AMDGPU_HW_IP_VCE] = 1, 401b1f2fecSChristian König [AMDGPU_HW_IP_UVD_ENC] = 1, 411b1f2fecSChristian König [AMDGPU_HW_IP_VCN_DEC] = 1, 421b1f2fecSChristian König [AMDGPU_HW_IP_VCN_ENC] = 1, 43f52c9643SAlex Deucher [AMDGPU_HW_IP_VCN_JPEG] = 1, 441b1f2fecSChristian König }; 451b1f2fecSChristian König 46c2636dc5SAndres Rodriguez static int amdgpu_ctx_priority_permit(struct drm_file *filp, 471b1f42d8SLucas Stach enum drm_sched_priority priority) 48c2636dc5SAndres Rodriguez { 49e2d732fdSLuben Tuikov if (priority < 0 || priority >= DRM_SCHED_PRIORITY_COUNT) 50977f7e10SNirmoy Das return -EINVAL; 51977f7e10SNirmoy Das 52c2636dc5SAndres Rodriguez /* NORMAL and below are accessible by everyone */ 531b1f42d8SLucas Stach if (priority <= DRM_SCHED_PRIORITY_NORMAL) 54c2636dc5SAndres Rodriguez return 0; 55c2636dc5SAndres Rodriguez 56c2636dc5SAndres Rodriguez if (capable(CAP_SYS_NICE)) 57c2636dc5SAndres Rodriguez return 0; 58c2636dc5SAndres Rodriguez 59c2636dc5SAndres Rodriguez if (drm_is_current_master(filp)) 60c2636dc5SAndres Rodriguez return 0; 61c2636dc5SAndres Rodriguez 62c2636dc5SAndres Rodriguez return -EACCES; 63c2636dc5SAndres Rodriguez } 64c2636dc5SAndres Rodriguez 6533abcb1fSNirmoy Das static enum gfx_pipe_priority amdgpu_ctx_sched_prio_to_compute_prio(enum drm_sched_priority prio) 6633abcb1fSNirmoy Das { 6733abcb1fSNirmoy Das switch (prio) { 68e2d732fdSLuben Tuikov case DRM_SCHED_PRIORITY_HIGH: 6933abcb1fSNirmoy Das case DRM_SCHED_PRIORITY_KERNEL: 7033abcb1fSNirmoy Das return AMDGPU_GFX_PIPE_PRIO_HIGH; 7133abcb1fSNirmoy Das default: 7233abcb1fSNirmoy Das return AMDGPU_GFX_PIPE_PRIO_NORMAL; 7333abcb1fSNirmoy Das } 7433abcb1fSNirmoy Das } 7533abcb1fSNirmoy Das 761c6d567bSNirmoy Das static unsigned int amdgpu_ctx_prio_sched_to_hw(struct amdgpu_device *adev, 771c6d567bSNirmoy Das enum drm_sched_priority prio, 781c6d567bSNirmoy Das u32 hw_ip) 791c6d567bSNirmoy Das { 801c6d567bSNirmoy Das unsigned int hw_prio; 811c6d567bSNirmoy Das 821c6d567bSNirmoy Das hw_prio = (hw_ip == AMDGPU_HW_IP_COMPUTE) ? 831c6d567bSNirmoy Das amdgpu_ctx_sched_prio_to_compute_prio(prio) : 841c6d567bSNirmoy Das AMDGPU_RING_PRIO_DEFAULT; 851c6d567bSNirmoy Das hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM); 861c6d567bSNirmoy Das if (adev->gpu_sched[hw_ip][hw_prio].num_scheds == 0) 871c6d567bSNirmoy Das hw_prio = AMDGPU_RING_PRIO_DEFAULT; 881c6d567bSNirmoy Das 891c6d567bSNirmoy Das return hw_prio; 901c6d567bSNirmoy Das } 911c6d567bSNirmoy Das 921c6d567bSNirmoy Das static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip, 931c6d567bSNirmoy Das const u32 ring) 94d38ceaf9SAlex Deucher { 95977f7e10SNirmoy Das struct amdgpu_device *adev = ctx->adev; 96977f7e10SNirmoy Das struct amdgpu_ctx_entity *entity; 97977f7e10SNirmoy Das struct drm_gpu_scheduler **scheds = NULL, *sched = NULL; 98977f7e10SNirmoy Das unsigned num_scheds = 0; 991c6d567bSNirmoy Das unsigned int hw_prio; 100977f7e10SNirmoy Das enum drm_sched_priority priority; 10147f38501SChristian König int r; 10247f38501SChristian König 103977f7e10SNirmoy Das entity = kcalloc(1, offsetof(typeof(*entity), fences[amdgpu_sched_jobs]), 1041b1f2fecSChristian König GFP_KERNEL); 105977f7e10SNirmoy Das if (!entity) 10663e3ab9aSNirmoy Das return -ENOMEM; 10763e3ab9aSNirmoy Das 1081b1f2fecSChristian König entity->sequence = 1; 109977f7e10SNirmoy Das priority = (ctx->override_priority == DRM_SCHED_PRIORITY_UNSET) ? 110977f7e10SNirmoy Das ctx->init_priority : ctx->override_priority; 1111c6d567bSNirmoy Das hw_prio = amdgpu_ctx_prio_sched_to_hw(adev, priority, hw_ip); 1121c6d567bSNirmoy Das 1131c6d567bSNirmoy Das hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM); 1141c6d567bSNirmoy Das scheds = adev->gpu_sched[hw_ip][hw_prio].sched; 1151c6d567bSNirmoy Das num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds; 1161c6d567bSNirmoy Das 117*bc21585fSNirmoy Das /* disable load balance if the hw engine retains context among dependent jobs */ 118*bc21585fSNirmoy Das if (hw_ip == AMDGPU_HW_IP_VCN_ENC || 119*bc21585fSNirmoy Das hw_ip == AMDGPU_HW_IP_VCN_DEC || 120*bc21585fSNirmoy Das hw_ip == AMDGPU_HW_IP_UVD_ENC || 121*bc21585fSNirmoy Das hw_ip == AMDGPU_HW_IP_UVD) { 1221c6d567bSNirmoy Das sched = drm_sched_pick_best(scheds, num_scheds); 1230a96afc7SLe Ma scheds = &sched; 124f880799dSNirmoy Das num_scheds = 1; 125845e6fdfSChristian König } 126845e6fdfSChristian König 127977f7e10SNirmoy Das r = drm_sched_entity_init(&entity->entity, priority, scheds, num_scheds, 128977f7e10SNirmoy Das &ctx->guilty); 1299cb7e5a9SChunming Zhou if (r) 130977f7e10SNirmoy Das goto error_free_entity; 131977f7e10SNirmoy Das 132977f7e10SNirmoy Das ctx->entities[hw_ip][ring] = entity; 133977f7e10SNirmoy Das return 0; 134977f7e10SNirmoy Das 135977f7e10SNirmoy Das error_free_entity: 136977f7e10SNirmoy Das kfree(entity); 137977f7e10SNirmoy Das 138977f7e10SNirmoy Das return r; 1399cb7e5a9SChunming Zhou } 1409cb7e5a9SChunming Zhou 141977f7e10SNirmoy Das static int amdgpu_ctx_init(struct amdgpu_device *adev, 142977f7e10SNirmoy Das enum drm_sched_priority priority, 143977f7e10SNirmoy Das struct drm_file *filp, 144977f7e10SNirmoy Das struct amdgpu_ctx *ctx) 145977f7e10SNirmoy Das { 146977f7e10SNirmoy Das int r; 147977f7e10SNirmoy Das 148977f7e10SNirmoy Das r = amdgpu_ctx_priority_permit(filp, priority); 149977f7e10SNirmoy Das if (r) 150977f7e10SNirmoy Das return r; 151977f7e10SNirmoy Das 152977f7e10SNirmoy Das memset(ctx, 0, sizeof(*ctx)); 153977f7e10SNirmoy Das 154977f7e10SNirmoy Das ctx->adev = adev; 155977f7e10SNirmoy Das 156977f7e10SNirmoy Das kref_init(&ctx->refcount); 157977f7e10SNirmoy Das spin_lock_init(&ctx->ring_lock); 158977f7e10SNirmoy Das mutex_init(&ctx->lock); 159977f7e10SNirmoy Das 160977f7e10SNirmoy Das ctx->reset_counter = atomic_read(&adev->gpu_reset_counter); 161977f7e10SNirmoy Das ctx->reset_counter_query = ctx->reset_counter; 162977f7e10SNirmoy Das ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter); 163977f7e10SNirmoy Das ctx->init_priority = priority; 164977f7e10SNirmoy Das ctx->override_priority = DRM_SCHED_PRIORITY_UNSET; 165977f7e10SNirmoy Das 1668ed8147aSHuang Rui return 0; 16763e3ab9aSNirmoy Das } 16863e3ab9aSNirmoy Das 169977f7e10SNirmoy Das static void amdgpu_ctx_fini_entity(struct amdgpu_ctx_entity *entity) 170977f7e10SNirmoy Das { 171977f7e10SNirmoy Das 172977f7e10SNirmoy Das int i; 173977f7e10SNirmoy Das 174977f7e10SNirmoy Das if (!entity) 175977f7e10SNirmoy Das return; 176977f7e10SNirmoy Das 177977f7e10SNirmoy Das for (i = 0; i < amdgpu_sched_jobs; ++i) 178977f7e10SNirmoy Das dma_fence_put(entity->fences[i]); 179977f7e10SNirmoy Das 180977f7e10SNirmoy Das kfree(entity); 1819cb7e5a9SChunming Zhou } 182d38ceaf9SAlex Deucher 1838ee3a52eSEmily Deng static void amdgpu_ctx_fini(struct kref *ref) 18447f38501SChristian König { 1858ee3a52eSEmily Deng struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount); 18647f38501SChristian König struct amdgpu_device *adev = ctx->adev; 18747f38501SChristian König unsigned i, j; 18847f38501SChristian König 189fe295b27SDave Airlie if (!adev) 190fe295b27SDave Airlie return; 191fe295b27SDave Airlie 192977f7e10SNirmoy Das for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 193977f7e10SNirmoy Das for (j = 0; j < AMDGPU_MAX_ENTITY_NUM; ++j) { 194977f7e10SNirmoy Das amdgpu_ctx_fini_entity(ctx->entities[i][j]); 195977f7e10SNirmoy Das ctx->entities[i][j] = NULL; 196977f7e10SNirmoy Das } 19763e3ab9aSNirmoy Das } 19863e3ab9aSNirmoy Das 1990ae94444SAndrey Grodzovsky mutex_destroy(&ctx->lock); 2008ee3a52eSEmily Deng kfree(ctx); 20147f38501SChristian König } 20247f38501SChristian König 2030d346a14SChristian König int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance, 2040d346a14SChristian König u32 ring, struct drm_sched_entity **entity) 205869a53d4SChristian König { 206977f7e10SNirmoy Das int r; 207977f7e10SNirmoy Das 2081b1f2fecSChristian König if (hw_ip >= AMDGPU_HW_IP_NUM) { 2091b1f2fecSChristian König DRM_ERROR("unknown HW IP type: %d\n", hw_ip); 2101b1f2fecSChristian König return -EINVAL; 2111b1f2fecSChristian König } 212869a53d4SChristian König 213869a53d4SChristian König /* Right now all IPs have only one instance - multiple rings. */ 214869a53d4SChristian König if (instance != 0) { 215869a53d4SChristian König DRM_DEBUG("invalid ip instance: %d\n", instance); 216869a53d4SChristian König return -EINVAL; 217869a53d4SChristian König } 218869a53d4SChristian König 2191b1f2fecSChristian König if (ring >= amdgpu_ctx_num_entities[hw_ip]) { 2201b1f2fecSChristian König DRM_DEBUG("invalid ring: %d %d\n", hw_ip, ring); 221869a53d4SChristian König return -EINVAL; 222869a53d4SChristian König } 223869a53d4SChristian König 224977f7e10SNirmoy Das if (ctx->entities[hw_ip][ring] == NULL) { 225977f7e10SNirmoy Das r = amdgpu_ctx_init_entity(ctx, hw_ip, ring); 226977f7e10SNirmoy Das if (r) 227977f7e10SNirmoy Das return r; 228977f7e10SNirmoy Das } 229977f7e10SNirmoy Das 230977f7e10SNirmoy Das *entity = &ctx->entities[hw_ip][ring]->entity; 231869a53d4SChristian König return 0; 232869a53d4SChristian König } 233869a53d4SChristian König 23447f38501SChristian König static int amdgpu_ctx_alloc(struct amdgpu_device *adev, 23547f38501SChristian König struct amdgpu_fpriv *fpriv, 236c2636dc5SAndres Rodriguez struct drm_file *filp, 2371b1f42d8SLucas Stach enum drm_sched_priority priority, 23847f38501SChristian König uint32_t *id) 23947f38501SChristian König { 24047f38501SChristian König struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; 24147f38501SChristian König struct amdgpu_ctx *ctx; 24247f38501SChristian König int r; 24347f38501SChristian König 24447f38501SChristian König ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 24547f38501SChristian König if (!ctx) 24647f38501SChristian König return -ENOMEM; 24747f38501SChristian König 24847f38501SChristian König mutex_lock(&mgr->lock); 24908d1bdd4SRex Zhu r = idr_alloc(&mgr->ctx_handles, ctx, 1, AMDGPU_VM_MAX_NUM_CTX, GFP_KERNEL); 25047f38501SChristian König if (r < 0) { 25147f38501SChristian König mutex_unlock(&mgr->lock); 25247f38501SChristian König kfree(ctx); 25347f38501SChristian König return r; 25447f38501SChristian König } 255c2636dc5SAndres Rodriguez 25647f38501SChristian König *id = (uint32_t)r; 257c2636dc5SAndres Rodriguez r = amdgpu_ctx_init(adev, priority, filp, ctx); 258c648ed7cSChunming Zhou if (r) { 259c648ed7cSChunming Zhou idr_remove(&mgr->ctx_handles, *id); 260c648ed7cSChunming Zhou *id = 0; 261c648ed7cSChunming Zhou kfree(ctx); 262c648ed7cSChunming Zhou } 26347f38501SChristian König mutex_unlock(&mgr->lock); 26447f38501SChristian König return r; 26547f38501SChristian König } 26647f38501SChristian König 26747f38501SChristian König static void amdgpu_ctx_do_release(struct kref *ref) 268d38ceaf9SAlex Deucher { 269d38ceaf9SAlex Deucher struct amdgpu_ctx *ctx; 270977f7e10SNirmoy Das u32 i, j; 271d38ceaf9SAlex Deucher 27247f38501SChristian König ctx = container_of(ref, struct amdgpu_ctx, refcount); 273977f7e10SNirmoy Das for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 274977f7e10SNirmoy Das for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { 275977f7e10SNirmoy Das if (!ctx->entities[i][j]) 276977f7e10SNirmoy Das continue; 27747f38501SChristian König 278977f7e10SNirmoy Das drm_sched_entity_destroy(&ctx->entities[i][j]->entity); 279977f7e10SNirmoy Das } 280977f7e10SNirmoy Das } 28147f38501SChristian König 2828ee3a52eSEmily Deng amdgpu_ctx_fini(ref); 28347f38501SChristian König } 28447f38501SChristian König 28547f38501SChristian König static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id) 28647f38501SChristian König { 28723ca0e4eSChunming Zhou struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; 28847f38501SChristian König struct amdgpu_ctx *ctx; 28947f38501SChristian König 2900147ee0fSMarek Olšák mutex_lock(&mgr->lock); 291d3e709e6SMatthew Wilcox ctx = idr_remove(&mgr->ctx_handles, id); 292d3e709e6SMatthew Wilcox if (ctx) 293f11358daSMarek Olšák kref_put(&ctx->refcount, amdgpu_ctx_do_release); 2940147ee0fSMarek Olšák mutex_unlock(&mgr->lock); 295d3e709e6SMatthew Wilcox return ctx ? 0 : -EINVAL; 296d38ceaf9SAlex Deucher } 297d38ceaf9SAlex Deucher 298d94aed5aSMarek Olšák static int amdgpu_ctx_query(struct amdgpu_device *adev, 299d94aed5aSMarek Olšák struct amdgpu_fpriv *fpriv, uint32_t id, 300d94aed5aSMarek Olšák union drm_amdgpu_ctx_out *out) 301d38ceaf9SAlex Deucher { 302d38ceaf9SAlex Deucher struct amdgpu_ctx *ctx; 30323ca0e4eSChunming Zhou struct amdgpu_ctx_mgr *mgr; 304d94aed5aSMarek Olšák unsigned reset_counter; 305d38ceaf9SAlex Deucher 30623ca0e4eSChunming Zhou if (!fpriv) 30723ca0e4eSChunming Zhou return -EINVAL; 30823ca0e4eSChunming Zhou 30923ca0e4eSChunming Zhou mgr = &fpriv->ctx_mgr; 3100147ee0fSMarek Olšák mutex_lock(&mgr->lock); 311d38ceaf9SAlex Deucher ctx = idr_find(&mgr->ctx_handles, id); 312d94aed5aSMarek Olšák if (!ctx) { 3130147ee0fSMarek Olšák mutex_unlock(&mgr->lock); 314d38ceaf9SAlex Deucher return -EINVAL; 315d38ceaf9SAlex Deucher } 316d38ceaf9SAlex Deucher 317d94aed5aSMarek Olšák /* TODO: these two are always zero */ 3180b492a4cSAlex Deucher out->state.flags = 0x0; 3190b492a4cSAlex Deucher out->state.hangs = 0x0; 320d94aed5aSMarek Olšák 321d94aed5aSMarek Olšák /* determine if a GPU reset has occured since the last call */ 322d94aed5aSMarek Olšák reset_counter = atomic_read(&adev->gpu_reset_counter); 323d94aed5aSMarek Olšák /* TODO: this should ideally return NO, GUILTY, or INNOCENT. */ 324668ca1b4SMonk Liu if (ctx->reset_counter_query == reset_counter) 325d94aed5aSMarek Olšák out->state.reset_status = AMDGPU_CTX_NO_RESET; 326d94aed5aSMarek Olšák else 327d94aed5aSMarek Olšák out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET; 328668ca1b4SMonk Liu ctx->reset_counter_query = reset_counter; 329d94aed5aSMarek Olšák 330d94aed5aSMarek Olšák mutex_unlock(&mgr->lock); 331d94aed5aSMarek Olšák return 0; 332d94aed5aSMarek Olšák } 333d94aed5aSMarek Olšák 334bc1b1bf6SMonk Liu static int amdgpu_ctx_query2(struct amdgpu_device *adev, 335bc1b1bf6SMonk Liu struct amdgpu_fpriv *fpriv, uint32_t id, 336bc1b1bf6SMonk Liu union drm_amdgpu_ctx_out *out) 337bc1b1bf6SMonk Liu { 338bc1b1bf6SMonk Liu struct amdgpu_ctx *ctx; 339bc1b1bf6SMonk Liu struct amdgpu_ctx_mgr *mgr; 34064cc5414SGuchun Chen unsigned long ras_counter; 341bc1b1bf6SMonk Liu 342bc1b1bf6SMonk Liu if (!fpriv) 343bc1b1bf6SMonk Liu return -EINVAL; 344bc1b1bf6SMonk Liu 345bc1b1bf6SMonk Liu mgr = &fpriv->ctx_mgr; 346bc1b1bf6SMonk Liu mutex_lock(&mgr->lock); 347bc1b1bf6SMonk Liu ctx = idr_find(&mgr->ctx_handles, id); 348bc1b1bf6SMonk Liu if (!ctx) { 349bc1b1bf6SMonk Liu mutex_unlock(&mgr->lock); 350bc1b1bf6SMonk Liu return -EINVAL; 351bc1b1bf6SMonk Liu } 352bc1b1bf6SMonk Liu 353bc1b1bf6SMonk Liu out->state.flags = 0x0; 354bc1b1bf6SMonk Liu out->state.hangs = 0x0; 355bc1b1bf6SMonk Liu 356bc1b1bf6SMonk Liu if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter)) 357bc1b1bf6SMonk Liu out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET; 358bc1b1bf6SMonk Liu 359bc1b1bf6SMonk Liu if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) 360bc1b1bf6SMonk Liu out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST; 361bc1b1bf6SMonk Liu 362bc1b1bf6SMonk Liu if (atomic_read(&ctx->guilty)) 363bc1b1bf6SMonk Liu out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY; 364bc1b1bf6SMonk Liu 365ae363a21Sxinhui pan /*query ue count*/ 366ae363a21Sxinhui pan ras_counter = amdgpu_ras_query_error_count(adev, false); 367ae363a21Sxinhui pan /*ras counter is monotonic increasing*/ 368ae363a21Sxinhui pan if (ras_counter != ctx->ras_counter_ue) { 369ae363a21Sxinhui pan out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_UE; 370ae363a21Sxinhui pan ctx->ras_counter_ue = ras_counter; 371ae363a21Sxinhui pan } 372ae363a21Sxinhui pan 373ae363a21Sxinhui pan /*query ce count*/ 374ae363a21Sxinhui pan ras_counter = amdgpu_ras_query_error_count(adev, true); 375ae363a21Sxinhui pan if (ras_counter != ctx->ras_counter_ce) { 376ae363a21Sxinhui pan out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_CE; 377ae363a21Sxinhui pan ctx->ras_counter_ce = ras_counter; 378ae363a21Sxinhui pan } 379ae363a21Sxinhui pan 380bc1b1bf6SMonk Liu mutex_unlock(&mgr->lock); 381bc1b1bf6SMonk Liu return 0; 382bc1b1bf6SMonk Liu } 383bc1b1bf6SMonk Liu 384d38ceaf9SAlex Deucher int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, 385d38ceaf9SAlex Deucher struct drm_file *filp) 386d38ceaf9SAlex Deucher { 387d38ceaf9SAlex Deucher int r; 388d38ceaf9SAlex Deucher uint32_t id; 3891b1f42d8SLucas Stach enum drm_sched_priority priority; 390d38ceaf9SAlex Deucher 391d38ceaf9SAlex Deucher union drm_amdgpu_ctx *args = data; 3921348969aSLuben Tuikov struct amdgpu_device *adev = drm_to_adev(dev); 393d38ceaf9SAlex Deucher struct amdgpu_fpriv *fpriv = filp->driver_priv; 394d38ceaf9SAlex Deucher 395d38ceaf9SAlex Deucher id = args->in.ctx_id; 3969af5e21dSLuben Tuikov r = amdgpu_to_sched_priority(args->in.priority, &priority); 397c2636dc5SAndres Rodriguez 398b6d8a439SAndres Rodriguez /* For backwards compatibility reasons, we need to accept 399b6d8a439SAndres Rodriguez * ioctls with garbage in the priority field */ 4009af5e21dSLuben Tuikov if (r == -EINVAL) 4011b1f42d8SLucas Stach priority = DRM_SCHED_PRIORITY_NORMAL; 402d38ceaf9SAlex Deucher 403d38ceaf9SAlex Deucher switch (args->in.op) { 404d38ceaf9SAlex Deucher case AMDGPU_CTX_OP_ALLOC_CTX: 405c2636dc5SAndres Rodriguez r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id); 406d38ceaf9SAlex Deucher args->out.alloc.ctx_id = id; 407d38ceaf9SAlex Deucher break; 408d38ceaf9SAlex Deucher case AMDGPU_CTX_OP_FREE_CTX: 40947f38501SChristian König r = amdgpu_ctx_free(fpriv, id); 410d38ceaf9SAlex Deucher break; 411d38ceaf9SAlex Deucher case AMDGPU_CTX_OP_QUERY_STATE: 412d94aed5aSMarek Olšák r = amdgpu_ctx_query(adev, fpriv, id, &args->out); 413d38ceaf9SAlex Deucher break; 414bc1b1bf6SMonk Liu case AMDGPU_CTX_OP_QUERY_STATE2: 415bc1b1bf6SMonk Liu r = amdgpu_ctx_query2(adev, fpriv, id, &args->out); 416bc1b1bf6SMonk Liu break; 417d38ceaf9SAlex Deucher default: 418d38ceaf9SAlex Deucher return -EINVAL; 419d38ceaf9SAlex Deucher } 420d38ceaf9SAlex Deucher 421d38ceaf9SAlex Deucher return r; 422d38ceaf9SAlex Deucher } 42366b3cf2aSJammy Zhou 42466b3cf2aSJammy Zhou struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id) 42566b3cf2aSJammy Zhou { 42666b3cf2aSJammy Zhou struct amdgpu_ctx *ctx; 42723ca0e4eSChunming Zhou struct amdgpu_ctx_mgr *mgr; 42823ca0e4eSChunming Zhou 42923ca0e4eSChunming Zhou if (!fpriv) 43023ca0e4eSChunming Zhou return NULL; 43123ca0e4eSChunming Zhou 43223ca0e4eSChunming Zhou mgr = &fpriv->ctx_mgr; 43366b3cf2aSJammy Zhou 43466b3cf2aSJammy Zhou mutex_lock(&mgr->lock); 43566b3cf2aSJammy Zhou ctx = idr_find(&mgr->ctx_handles, id); 43666b3cf2aSJammy Zhou if (ctx) 43766b3cf2aSJammy Zhou kref_get(&ctx->refcount); 43866b3cf2aSJammy Zhou mutex_unlock(&mgr->lock); 43966b3cf2aSJammy Zhou return ctx; 44066b3cf2aSJammy Zhou } 44166b3cf2aSJammy Zhou 44266b3cf2aSJammy Zhou int amdgpu_ctx_put(struct amdgpu_ctx *ctx) 44366b3cf2aSJammy Zhou { 44466b3cf2aSJammy Zhou if (ctx == NULL) 44566b3cf2aSJammy Zhou return -EINVAL; 44666b3cf2aSJammy Zhou 44766b3cf2aSJammy Zhou kref_put(&ctx->refcount, amdgpu_ctx_do_release); 44866b3cf2aSJammy Zhou return 0; 44966b3cf2aSJammy Zhou } 45021c16bf6SChristian König 45185eff200SChristian König void amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, 4520d346a14SChristian König struct drm_sched_entity *entity, 4530d346a14SChristian König struct dma_fence *fence, uint64_t* handle) 45421c16bf6SChristian König { 4551b1f2fecSChristian König struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); 4561b1f2fecSChristian König uint64_t seq = centity->sequence; 457f54d1867SChris Wilson struct dma_fence *other = NULL; 4580d346a14SChristian König unsigned idx = 0; 45921c16bf6SChristian König 4605b011235SChunming Zhou idx = seq & (amdgpu_sched_jobs - 1); 4611b1f2fecSChristian König other = centity->fences[idx]; 4620ae94444SAndrey Grodzovsky if (other) 4630ae94444SAndrey Grodzovsky BUG_ON(!dma_fence_is_signaled(other)); 46421c16bf6SChristian König 465f54d1867SChris Wilson dma_fence_get(fence); 46621c16bf6SChristian König 46721c16bf6SChristian König spin_lock(&ctx->ring_lock); 4681b1f2fecSChristian König centity->fences[idx] = fence; 4691b1f2fecSChristian König centity->sequence++; 47021c16bf6SChristian König spin_unlock(&ctx->ring_lock); 47121c16bf6SChristian König 472f54d1867SChris Wilson dma_fence_put(other); 4730d346a14SChristian König if (handle) 4740d346a14SChristian König *handle = seq; 47521c16bf6SChristian König } 47621c16bf6SChristian König 477f54d1867SChris Wilson struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, 4780d346a14SChristian König struct drm_sched_entity *entity, 4790d346a14SChristian König uint64_t seq) 48021c16bf6SChristian König { 4811b1f2fecSChristian König struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); 482f54d1867SChris Wilson struct dma_fence *fence; 48321c16bf6SChristian König 48421c16bf6SChristian König spin_lock(&ctx->ring_lock); 485b43a9a7eSChunming Zhou 486d7b1eeb2SMonk Liu if (seq == ~0ull) 4871b1f2fecSChristian König seq = centity->sequence - 1; 488d7b1eeb2SMonk Liu 4891b1f2fecSChristian König if (seq >= centity->sequence) { 49021c16bf6SChristian König spin_unlock(&ctx->ring_lock); 49121c16bf6SChristian König return ERR_PTR(-EINVAL); 49221c16bf6SChristian König } 49321c16bf6SChristian König 494b43a9a7eSChunming Zhou 4951b1f2fecSChristian König if (seq + amdgpu_sched_jobs < centity->sequence) { 49621c16bf6SChristian König spin_unlock(&ctx->ring_lock); 49721c16bf6SChristian König return NULL; 49821c16bf6SChristian König } 49921c16bf6SChristian König 5001b1f2fecSChristian König fence = dma_fence_get(centity->fences[seq & (amdgpu_sched_jobs - 1)]); 50121c16bf6SChristian König spin_unlock(&ctx->ring_lock); 50221c16bf6SChristian König 50321c16bf6SChristian König return fence; 50421c16bf6SChristian König } 505efd4ccb5SChristian König 5062316a86bSNirmoy Das static void amdgpu_ctx_set_entity_priority(struct amdgpu_ctx *ctx, 5072316a86bSNirmoy Das struct amdgpu_ctx_entity *aentity, 5082316a86bSNirmoy Das int hw_ip, 5092316a86bSNirmoy Das enum drm_sched_priority priority) 5102316a86bSNirmoy Das { 5112316a86bSNirmoy Das struct amdgpu_device *adev = ctx->adev; 5121c6d567bSNirmoy Das unsigned int hw_prio; 5132316a86bSNirmoy Das struct drm_gpu_scheduler **scheds = NULL; 5142316a86bSNirmoy Das unsigned num_scheds; 5152316a86bSNirmoy Das 5162316a86bSNirmoy Das /* set sw priority */ 5172316a86bSNirmoy Das drm_sched_entity_set_priority(&aentity->entity, priority); 5182316a86bSNirmoy Das 5192316a86bSNirmoy Das /* set hw priority */ 5202316a86bSNirmoy Das if (hw_ip == AMDGPU_HW_IP_COMPUTE) { 5211c6d567bSNirmoy Das hw_prio = amdgpu_ctx_prio_sched_to_hw(adev, priority, 5221c6d567bSNirmoy Das AMDGPU_HW_IP_COMPUTE); 5231c6d567bSNirmoy Das hw_prio = array_index_nospec(hw_prio, AMDGPU_RING_PRIO_MAX); 5241c6d567bSNirmoy Das scheds = adev->gpu_sched[hw_ip][hw_prio].sched; 5251c6d567bSNirmoy Das num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds; 5262316a86bSNirmoy Das drm_sched_entity_modify_sched(&aentity->entity, scheds, 5272316a86bSNirmoy Das num_scheds); 5282316a86bSNirmoy Das } 5292316a86bSNirmoy Das } 5302316a86bSNirmoy Das 531c23be4aeSAndres Rodriguez void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx, 5321b1f42d8SLucas Stach enum drm_sched_priority priority) 533c23be4aeSAndres Rodriguez { 5341b1f42d8SLucas Stach enum drm_sched_priority ctx_prio; 535977f7e10SNirmoy Das unsigned i, j; 536c23be4aeSAndres Rodriguez 537c23be4aeSAndres Rodriguez ctx->override_priority = priority; 538c23be4aeSAndres Rodriguez 5391b1f42d8SLucas Stach ctx_prio = (ctx->override_priority == DRM_SCHED_PRIORITY_UNSET) ? 540c23be4aeSAndres Rodriguez ctx->init_priority : ctx->override_priority; 541977f7e10SNirmoy Das for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 542977f7e10SNirmoy Das for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { 543977f7e10SNirmoy Das if (!ctx->entities[i][j]) 544977f7e10SNirmoy Das continue; 545c23be4aeSAndres Rodriguez 5462316a86bSNirmoy Das amdgpu_ctx_set_entity_priority(ctx, ctx->entities[i][j], 5472316a86bSNirmoy Das i, ctx_prio); 548c23be4aeSAndres Rodriguez } 549c23be4aeSAndres Rodriguez } 550977f7e10SNirmoy Das } 551c23be4aeSAndres Rodriguez 5520d346a14SChristian König int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, 5530d346a14SChristian König struct drm_sched_entity *entity) 5540ae94444SAndrey Grodzovsky { 5551b1f2fecSChristian König struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); 55675e1cafdSChristian König struct dma_fence *other; 55775e1cafdSChristian König unsigned idx; 55875e1cafdSChristian König long r; 5590ae94444SAndrey Grodzovsky 56075e1cafdSChristian König spin_lock(&ctx->ring_lock); 56175e1cafdSChristian König idx = centity->sequence & (amdgpu_sched_jobs - 1); 56275e1cafdSChristian König other = dma_fence_get(centity->fences[idx]); 56375e1cafdSChristian König spin_unlock(&ctx->ring_lock); 56475e1cafdSChristian König 56575e1cafdSChristian König if (!other) 56675e1cafdSChristian König return 0; 56775e1cafdSChristian König 568719a39a1SAndrey Grodzovsky r = dma_fence_wait(other, true); 56975e1cafdSChristian König if (r < 0 && r != -ERESTARTSYS) 5700ae94444SAndrey Grodzovsky DRM_ERROR("Error (%ld) waiting for fence!\n", r); 571719a39a1SAndrey Grodzovsky 57275e1cafdSChristian König dma_fence_put(other); 5730ae94444SAndrey Grodzovsky return r; 5740ae94444SAndrey Grodzovsky } 5750ae94444SAndrey Grodzovsky 576efd4ccb5SChristian König void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr) 577efd4ccb5SChristian König { 578efd4ccb5SChristian König mutex_init(&mgr->lock); 579efd4ccb5SChristian König idr_init(&mgr->ctx_handles); 580efd4ccb5SChristian König } 581efd4ccb5SChristian König 58256753e73SChristian König long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout) 5838ee3a52eSEmily Deng { 5848ee3a52eSEmily Deng struct amdgpu_ctx *ctx; 5858ee3a52eSEmily Deng struct idr *idp; 586977f7e10SNirmoy Das uint32_t id, i, j; 5878ee3a52eSEmily Deng 5888ee3a52eSEmily Deng idp = &mgr->ctx_handles; 5898ee3a52eSEmily Deng 59048ad368aSAndrey Grodzovsky mutex_lock(&mgr->lock); 5918ee3a52eSEmily Deng idr_for_each_entry(idp, ctx, id) { 592977f7e10SNirmoy Das for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 593977f7e10SNirmoy Das for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { 5941b1f2fecSChristian König struct drm_sched_entity *entity; 59520b6b788SAndrey Grodzovsky 596977f7e10SNirmoy Das if (!ctx->entities[i][j]) 597977f7e10SNirmoy Das continue; 598977f7e10SNirmoy Das 599977f7e10SNirmoy Das entity = &ctx->entities[i][j]->entity; 60056753e73SChristian König timeout = drm_sched_entity_flush(entity, timeout); 6018ee3a52eSEmily Deng } 6028ee3a52eSEmily Deng } 603977f7e10SNirmoy Das } 60448ad368aSAndrey Grodzovsky mutex_unlock(&mgr->lock); 60556753e73SChristian König return timeout; 60620b6b788SAndrey Grodzovsky } 6078ee3a52eSEmily Deng 608c49d8280SAndrey Grodzovsky void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr) 6098ee3a52eSEmily Deng { 6108ee3a52eSEmily Deng struct amdgpu_ctx *ctx; 6118ee3a52eSEmily Deng struct idr *idp; 612977f7e10SNirmoy Das uint32_t id, i, j; 6138ee3a52eSEmily Deng 6148ee3a52eSEmily Deng idp = &mgr->ctx_handles; 6158ee3a52eSEmily Deng 6168ee3a52eSEmily Deng idr_for_each_entry(idp, ctx, id) { 6171b1f2fecSChristian König if (kref_read(&ctx->refcount) != 1) { 6188ee3a52eSEmily Deng DRM_ERROR("ctx %p is still alive\n", ctx); 6191b1f2fecSChristian König continue; 6208ee3a52eSEmily Deng } 6211b1f2fecSChristian König 622977f7e10SNirmoy Das for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 623977f7e10SNirmoy Das for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { 624977f7e10SNirmoy Das struct drm_sched_entity *entity; 625977f7e10SNirmoy Das 626977f7e10SNirmoy Das if (!ctx->entities[i][j]) 627977f7e10SNirmoy Das continue; 628977f7e10SNirmoy Das 629977f7e10SNirmoy Das entity = &ctx->entities[i][j]->entity; 630977f7e10SNirmoy Das drm_sched_entity_fini(entity); 631977f7e10SNirmoy Das } 632977f7e10SNirmoy Das } 6338ee3a52eSEmily Deng } 63420b6b788SAndrey Grodzovsky } 6358ee3a52eSEmily Deng 636efd4ccb5SChristian König void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr) 637efd4ccb5SChristian König { 638efd4ccb5SChristian König struct amdgpu_ctx *ctx; 639efd4ccb5SChristian König struct idr *idp; 640efd4ccb5SChristian König uint32_t id; 641efd4ccb5SChristian König 642c49d8280SAndrey Grodzovsky amdgpu_ctx_mgr_entity_fini(mgr); 6438ee3a52eSEmily Deng 644efd4ccb5SChristian König idp = &mgr->ctx_handles; 645efd4ccb5SChristian König 646efd4ccb5SChristian König idr_for_each_entry(idp, ctx, id) { 6478ee3a52eSEmily Deng if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1) 648efd4ccb5SChristian König DRM_ERROR("ctx %p is still alive\n", ctx); 649efd4ccb5SChristian König } 650efd4ccb5SChristian König 651efd4ccb5SChristian König idr_destroy(&mgr->ctx_handles); 652efd4ccb5SChristian König mutex_destroy(&mgr->lock); 653efd4ccb5SChristian König } 654