1 /* 2 * Copyright 2015 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: monk liu <monk.liu@amd.com> 23 */ 24 25 #include <drm/drm_auth.h> 26 #include "amdgpu.h" 27 #include "amdgpu_sched.h" 28 #include "amdgpu_ras.h" 29 #include <linux/nospec.h> 30 31 #define to_amdgpu_ctx_entity(e) \ 32 container_of((e), struct amdgpu_ctx_entity, entity) 33 34 const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM] = { 35 [AMDGPU_HW_IP_GFX] = 1, 36 [AMDGPU_HW_IP_COMPUTE] = 4, 37 [AMDGPU_HW_IP_DMA] = 2, 38 [AMDGPU_HW_IP_UVD] = 1, 39 [AMDGPU_HW_IP_VCE] = 1, 40 [AMDGPU_HW_IP_UVD_ENC] = 1, 41 [AMDGPU_HW_IP_VCN_DEC] = 1, 42 [AMDGPU_HW_IP_VCN_ENC] = 1, 43 [AMDGPU_HW_IP_VCN_JPEG] = 1, 44 }; 45 46 bool amdgpu_ctx_priority_is_valid(int32_t ctx_prio) 47 { 48 switch (ctx_prio) { 49 case AMDGPU_CTX_PRIORITY_UNSET: 50 case AMDGPU_CTX_PRIORITY_VERY_LOW: 51 case AMDGPU_CTX_PRIORITY_LOW: 52 case AMDGPU_CTX_PRIORITY_NORMAL: 53 case AMDGPU_CTX_PRIORITY_HIGH: 54 case AMDGPU_CTX_PRIORITY_VERY_HIGH: 55 return true; 56 default: 57 return false; 58 } 59 } 60 61 static enum drm_sched_priority 62 amdgpu_ctx_to_drm_sched_prio(int32_t ctx_prio) 63 { 64 switch (ctx_prio) { 65 case AMDGPU_CTX_PRIORITY_UNSET: 66 return DRM_SCHED_PRIORITY_UNSET; 67 68 case AMDGPU_CTX_PRIORITY_VERY_LOW: 69 return DRM_SCHED_PRIORITY_MIN; 70 71 case AMDGPU_CTX_PRIORITY_LOW: 72 return DRM_SCHED_PRIORITY_MIN; 73 74 case AMDGPU_CTX_PRIORITY_NORMAL: 75 return DRM_SCHED_PRIORITY_NORMAL; 76 77 case AMDGPU_CTX_PRIORITY_HIGH: 78 return DRM_SCHED_PRIORITY_HIGH; 79 80 case AMDGPU_CTX_PRIORITY_VERY_HIGH: 81 return DRM_SCHED_PRIORITY_HIGH; 82 83 /* This should not happen as we sanitized userspace provided priority 84 * already, WARN if this happens. 85 */ 86 default: 87 WARN(1, "Invalid context priority %d\n", ctx_prio); 88 return DRM_SCHED_PRIORITY_NORMAL; 89 } 90 91 } 92 93 static int amdgpu_ctx_priority_permit(struct drm_file *filp, 94 int32_t priority) 95 { 96 if (!amdgpu_ctx_priority_is_valid(priority)) 97 return -EINVAL; 98 99 /* NORMAL and below are accessible by everyone */ 100 if (priority <= AMDGPU_CTX_PRIORITY_NORMAL) 101 return 0; 102 103 if (capable(CAP_SYS_NICE)) 104 return 0; 105 106 if (drm_is_current_master(filp)) 107 return 0; 108 109 return -EACCES; 110 } 111 112 static enum amdgpu_gfx_pipe_priority amdgpu_ctx_prio_to_compute_prio(int32_t prio) 113 { 114 switch (prio) { 115 case AMDGPU_CTX_PRIORITY_HIGH: 116 case AMDGPU_CTX_PRIORITY_VERY_HIGH: 117 return AMDGPU_GFX_PIPE_PRIO_HIGH; 118 default: 119 return AMDGPU_GFX_PIPE_PRIO_NORMAL; 120 } 121 } 122 123 static enum amdgpu_ring_priority_level amdgpu_ctx_sched_prio_to_ring_prio(int32_t prio) 124 { 125 switch (prio) { 126 case AMDGPU_CTX_PRIORITY_HIGH: 127 return AMDGPU_RING_PRIO_1; 128 case AMDGPU_CTX_PRIORITY_VERY_HIGH: 129 return AMDGPU_RING_PRIO_2; 130 default: 131 return AMDGPU_RING_PRIO_0; 132 } 133 } 134 135 static unsigned int amdgpu_ctx_get_hw_prio(struct amdgpu_ctx *ctx, u32 hw_ip) 136 { 137 struct amdgpu_device *adev = ctx->adev; 138 int32_t ctx_prio; 139 unsigned int hw_prio; 140 141 ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ? 142 ctx->init_priority : ctx->override_priority; 143 144 switch (hw_ip) { 145 case AMDGPU_HW_IP_COMPUTE: 146 hw_prio = amdgpu_ctx_prio_to_compute_prio(ctx_prio); 147 break; 148 case AMDGPU_HW_IP_VCE: 149 case AMDGPU_HW_IP_VCN_ENC: 150 hw_prio = amdgpu_ctx_sched_prio_to_ring_prio(ctx_prio); 151 break; 152 default: 153 hw_prio = AMDGPU_RING_PRIO_DEFAULT; 154 break; 155 } 156 157 hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM); 158 if (adev->gpu_sched[hw_ip][hw_prio].num_scheds == 0) 159 hw_prio = AMDGPU_RING_PRIO_DEFAULT; 160 161 return hw_prio; 162 } 163 164 165 static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip, 166 const u32 ring) 167 { 168 struct amdgpu_device *adev = ctx->adev; 169 struct amdgpu_ctx_entity *entity; 170 struct drm_gpu_scheduler **scheds = NULL, *sched = NULL; 171 unsigned num_scheds = 0; 172 int32_t ctx_prio; 173 unsigned int hw_prio; 174 enum drm_sched_priority drm_prio; 175 int r; 176 177 entity = kzalloc(struct_size(entity, fences, amdgpu_sched_jobs), 178 GFP_KERNEL); 179 if (!entity) 180 return -ENOMEM; 181 182 ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ? 183 ctx->init_priority : ctx->override_priority; 184 entity->sequence = 1; 185 hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip); 186 drm_prio = amdgpu_ctx_to_drm_sched_prio(ctx_prio); 187 188 hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM); 189 scheds = adev->gpu_sched[hw_ip][hw_prio].sched; 190 num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds; 191 192 /* disable load balance if the hw engine retains context among dependent jobs */ 193 if (hw_ip == AMDGPU_HW_IP_VCN_ENC || 194 hw_ip == AMDGPU_HW_IP_VCN_DEC || 195 hw_ip == AMDGPU_HW_IP_UVD_ENC || 196 hw_ip == AMDGPU_HW_IP_UVD) { 197 sched = drm_sched_pick_best(scheds, num_scheds); 198 scheds = &sched; 199 num_scheds = 1; 200 } 201 202 r = drm_sched_entity_init(&entity->entity, drm_prio, scheds, num_scheds, 203 &ctx->guilty); 204 if (r) 205 goto error_free_entity; 206 207 ctx->entities[hw_ip][ring] = entity; 208 return 0; 209 210 error_free_entity: 211 kfree(entity); 212 213 return r; 214 } 215 216 static int amdgpu_ctx_init(struct amdgpu_device *adev, 217 int32_t priority, 218 struct drm_file *filp, 219 struct amdgpu_ctx *ctx) 220 { 221 int r; 222 223 r = amdgpu_ctx_priority_permit(filp, priority); 224 if (r) 225 return r; 226 227 memset(ctx, 0, sizeof(*ctx)); 228 229 ctx->adev = adev; 230 231 kref_init(&ctx->refcount); 232 spin_lock_init(&ctx->ring_lock); 233 mutex_init(&ctx->lock); 234 235 ctx->reset_counter = atomic_read(&adev->gpu_reset_counter); 236 ctx->reset_counter_query = ctx->reset_counter; 237 ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter); 238 ctx->init_priority = priority; 239 ctx->override_priority = AMDGPU_CTX_PRIORITY_UNSET; 240 ctx->stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE; 241 242 return 0; 243 } 244 245 static void amdgpu_ctx_fini_entity(struct amdgpu_ctx_entity *entity) 246 { 247 248 int i; 249 250 if (!entity) 251 return; 252 253 for (i = 0; i < amdgpu_sched_jobs; ++i) 254 dma_fence_put(entity->fences[i]); 255 256 kfree(entity); 257 } 258 259 static int amdgpu_ctx_get_stable_pstate(struct amdgpu_ctx *ctx, 260 u32 *stable_pstate) 261 { 262 struct amdgpu_device *adev = ctx->adev; 263 enum amd_dpm_forced_level current_level; 264 265 if (!ctx) 266 return -EINVAL; 267 268 current_level = amdgpu_dpm_get_performance_level(adev); 269 270 switch (current_level) { 271 case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD: 272 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_STANDARD; 273 break; 274 case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK: 275 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK; 276 break; 277 case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK: 278 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK; 279 break; 280 case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK: 281 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_PEAK; 282 break; 283 default: 284 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE; 285 break; 286 } 287 return 0; 288 } 289 290 static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, 291 u32 stable_pstate) 292 { 293 struct amdgpu_device *adev = ctx->adev; 294 enum amd_dpm_forced_level level; 295 int r; 296 297 if (!ctx) 298 return -EINVAL; 299 300 mutex_lock(&adev->pm.stable_pstate_ctx_lock); 301 if (adev->pm.stable_pstate_ctx && adev->pm.stable_pstate_ctx != ctx) { 302 r = -EBUSY; 303 goto done; 304 } 305 306 switch (stable_pstate) { 307 case AMDGPU_CTX_STABLE_PSTATE_NONE: 308 level = AMD_DPM_FORCED_LEVEL_AUTO; 309 break; 310 case AMDGPU_CTX_STABLE_PSTATE_STANDARD: 311 level = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD; 312 break; 313 case AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK: 314 level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK; 315 break; 316 case AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK: 317 level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK; 318 break; 319 case AMDGPU_CTX_STABLE_PSTATE_PEAK: 320 level = AMD_DPM_FORCED_LEVEL_PROFILE_PEAK; 321 break; 322 default: 323 r = -EINVAL; 324 goto done; 325 } 326 327 r = amdgpu_dpm_force_performance_level(adev, level); 328 329 if (level == AMD_DPM_FORCED_LEVEL_AUTO) 330 adev->pm.stable_pstate_ctx = NULL; 331 else 332 adev->pm.stable_pstate_ctx = ctx; 333 done: 334 mutex_unlock(&adev->pm.stable_pstate_ctx_lock); 335 336 return r; 337 } 338 339 static void amdgpu_ctx_fini(struct kref *ref) 340 { 341 struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount); 342 struct amdgpu_device *adev = ctx->adev; 343 unsigned i, j; 344 345 if (!adev) 346 return; 347 348 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 349 for (j = 0; j < AMDGPU_MAX_ENTITY_NUM; ++j) { 350 amdgpu_ctx_fini_entity(ctx->entities[i][j]); 351 ctx->entities[i][j] = NULL; 352 } 353 } 354 amdgpu_ctx_set_stable_pstate(ctx, AMDGPU_CTX_STABLE_PSTATE_NONE); 355 mutex_destroy(&ctx->lock); 356 kfree(ctx); 357 } 358 359 int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance, 360 u32 ring, struct drm_sched_entity **entity) 361 { 362 int r; 363 364 if (hw_ip >= AMDGPU_HW_IP_NUM) { 365 DRM_ERROR("unknown HW IP type: %d\n", hw_ip); 366 return -EINVAL; 367 } 368 369 /* Right now all IPs have only one instance - multiple rings. */ 370 if (instance != 0) { 371 DRM_DEBUG("invalid ip instance: %d\n", instance); 372 return -EINVAL; 373 } 374 375 if (ring >= amdgpu_ctx_num_entities[hw_ip]) { 376 DRM_DEBUG("invalid ring: %d %d\n", hw_ip, ring); 377 return -EINVAL; 378 } 379 380 if (ctx->entities[hw_ip][ring] == NULL) { 381 r = amdgpu_ctx_init_entity(ctx, hw_ip, ring); 382 if (r) 383 return r; 384 } 385 386 *entity = &ctx->entities[hw_ip][ring]->entity; 387 return 0; 388 } 389 390 static int amdgpu_ctx_alloc(struct amdgpu_device *adev, 391 struct amdgpu_fpriv *fpriv, 392 struct drm_file *filp, 393 int32_t priority, 394 uint32_t *id) 395 { 396 struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; 397 struct amdgpu_ctx *ctx; 398 int r; 399 400 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 401 if (!ctx) 402 return -ENOMEM; 403 404 mutex_lock(&mgr->lock); 405 r = idr_alloc(&mgr->ctx_handles, ctx, 1, AMDGPU_VM_MAX_NUM_CTX, GFP_KERNEL); 406 if (r < 0) { 407 mutex_unlock(&mgr->lock); 408 kfree(ctx); 409 return r; 410 } 411 412 *id = (uint32_t)r; 413 r = amdgpu_ctx_init(adev, priority, filp, ctx); 414 if (r) { 415 idr_remove(&mgr->ctx_handles, *id); 416 *id = 0; 417 kfree(ctx); 418 } 419 mutex_unlock(&mgr->lock); 420 return r; 421 } 422 423 static void amdgpu_ctx_do_release(struct kref *ref) 424 { 425 struct amdgpu_ctx *ctx; 426 u32 i, j; 427 428 ctx = container_of(ref, struct amdgpu_ctx, refcount); 429 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 430 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { 431 if (!ctx->entities[i][j]) 432 continue; 433 434 drm_sched_entity_destroy(&ctx->entities[i][j]->entity); 435 } 436 } 437 438 amdgpu_ctx_fini(ref); 439 } 440 441 static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id) 442 { 443 struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; 444 struct amdgpu_ctx *ctx; 445 446 mutex_lock(&mgr->lock); 447 ctx = idr_remove(&mgr->ctx_handles, id); 448 if (ctx) 449 kref_put(&ctx->refcount, amdgpu_ctx_do_release); 450 mutex_unlock(&mgr->lock); 451 return ctx ? 0 : -EINVAL; 452 } 453 454 static int amdgpu_ctx_query(struct amdgpu_device *adev, 455 struct amdgpu_fpriv *fpriv, uint32_t id, 456 union drm_amdgpu_ctx_out *out) 457 { 458 struct amdgpu_ctx *ctx; 459 struct amdgpu_ctx_mgr *mgr; 460 unsigned reset_counter; 461 462 if (!fpriv) 463 return -EINVAL; 464 465 mgr = &fpriv->ctx_mgr; 466 mutex_lock(&mgr->lock); 467 ctx = idr_find(&mgr->ctx_handles, id); 468 if (!ctx) { 469 mutex_unlock(&mgr->lock); 470 return -EINVAL; 471 } 472 473 /* TODO: these two are always zero */ 474 out->state.flags = 0x0; 475 out->state.hangs = 0x0; 476 477 /* determine if a GPU reset has occured since the last call */ 478 reset_counter = atomic_read(&adev->gpu_reset_counter); 479 /* TODO: this should ideally return NO, GUILTY, or INNOCENT. */ 480 if (ctx->reset_counter_query == reset_counter) 481 out->state.reset_status = AMDGPU_CTX_NO_RESET; 482 else 483 out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET; 484 ctx->reset_counter_query = reset_counter; 485 486 mutex_unlock(&mgr->lock); 487 return 0; 488 } 489 490 #define AMDGPU_RAS_COUNTE_DELAY_MS 3000 491 492 static int amdgpu_ctx_query2(struct amdgpu_device *adev, 493 struct amdgpu_fpriv *fpriv, uint32_t id, 494 union drm_amdgpu_ctx_out *out) 495 { 496 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 497 struct amdgpu_ctx *ctx; 498 struct amdgpu_ctx_mgr *mgr; 499 500 if (!fpriv) 501 return -EINVAL; 502 503 mgr = &fpriv->ctx_mgr; 504 mutex_lock(&mgr->lock); 505 ctx = idr_find(&mgr->ctx_handles, id); 506 if (!ctx) { 507 mutex_unlock(&mgr->lock); 508 return -EINVAL; 509 } 510 511 out->state.flags = 0x0; 512 out->state.hangs = 0x0; 513 514 if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter)) 515 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET; 516 517 if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) 518 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST; 519 520 if (atomic_read(&ctx->guilty)) 521 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY; 522 523 if (adev->ras_enabled && con) { 524 /* Return the cached values in O(1), 525 * and schedule delayed work to cache 526 * new vaues. 527 */ 528 int ce_count, ue_count; 529 530 ce_count = atomic_read(&con->ras_ce_count); 531 ue_count = atomic_read(&con->ras_ue_count); 532 533 if (ce_count != ctx->ras_counter_ce) { 534 ctx->ras_counter_ce = ce_count; 535 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_CE; 536 } 537 538 if (ue_count != ctx->ras_counter_ue) { 539 ctx->ras_counter_ue = ue_count; 540 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_UE; 541 } 542 543 schedule_delayed_work(&con->ras_counte_delay_work, 544 msecs_to_jiffies(AMDGPU_RAS_COUNTE_DELAY_MS)); 545 } 546 547 mutex_unlock(&mgr->lock); 548 return 0; 549 } 550 551 552 553 static int amdgpu_ctx_stable_pstate(struct amdgpu_device *adev, 554 struct amdgpu_fpriv *fpriv, uint32_t id, 555 bool set, u32 *stable_pstate) 556 { 557 struct amdgpu_ctx *ctx; 558 struct amdgpu_ctx_mgr *mgr; 559 int r; 560 561 if (!fpriv) 562 return -EINVAL; 563 564 mgr = &fpriv->ctx_mgr; 565 mutex_lock(&mgr->lock); 566 ctx = idr_find(&mgr->ctx_handles, id); 567 if (!ctx) { 568 mutex_unlock(&mgr->lock); 569 return -EINVAL; 570 } 571 572 if (set) 573 r = amdgpu_ctx_set_stable_pstate(ctx, *stable_pstate); 574 else 575 r = amdgpu_ctx_get_stable_pstate(ctx, stable_pstate); 576 577 mutex_unlock(&mgr->lock); 578 return r; 579 } 580 581 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, 582 struct drm_file *filp) 583 { 584 int r; 585 uint32_t id, stable_pstate; 586 int32_t priority; 587 588 union drm_amdgpu_ctx *args = data; 589 struct amdgpu_device *adev = drm_to_adev(dev); 590 struct amdgpu_fpriv *fpriv = filp->driver_priv; 591 592 id = args->in.ctx_id; 593 priority = args->in.priority; 594 595 /* For backwards compatibility reasons, we need to accept 596 * ioctls with garbage in the priority field */ 597 if (!amdgpu_ctx_priority_is_valid(priority)) 598 priority = AMDGPU_CTX_PRIORITY_NORMAL; 599 600 switch (args->in.op) { 601 case AMDGPU_CTX_OP_ALLOC_CTX: 602 r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id); 603 args->out.alloc.ctx_id = id; 604 break; 605 case AMDGPU_CTX_OP_FREE_CTX: 606 r = amdgpu_ctx_free(fpriv, id); 607 break; 608 case AMDGPU_CTX_OP_QUERY_STATE: 609 r = amdgpu_ctx_query(adev, fpriv, id, &args->out); 610 break; 611 case AMDGPU_CTX_OP_QUERY_STATE2: 612 r = amdgpu_ctx_query2(adev, fpriv, id, &args->out); 613 break; 614 case AMDGPU_CTX_OP_GET_STABLE_PSTATE: 615 if (args->in.flags) 616 return -EINVAL; 617 r = amdgpu_ctx_stable_pstate(adev, fpriv, id, false, &stable_pstate); 618 args->out.pstate.flags = stable_pstate; 619 break; 620 case AMDGPU_CTX_OP_SET_STABLE_PSTATE: 621 if (args->in.flags & ~AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK) 622 return -EINVAL; 623 stable_pstate = args->in.flags & AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK; 624 if (stable_pstate > AMDGPU_CTX_STABLE_PSTATE_PEAK) 625 return -EINVAL; 626 r = amdgpu_ctx_stable_pstate(adev, fpriv, id, true, &stable_pstate); 627 break; 628 default: 629 return -EINVAL; 630 } 631 632 return r; 633 } 634 635 struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id) 636 { 637 struct amdgpu_ctx *ctx; 638 struct amdgpu_ctx_mgr *mgr; 639 640 if (!fpriv) 641 return NULL; 642 643 mgr = &fpriv->ctx_mgr; 644 645 mutex_lock(&mgr->lock); 646 ctx = idr_find(&mgr->ctx_handles, id); 647 if (ctx) 648 kref_get(&ctx->refcount); 649 mutex_unlock(&mgr->lock); 650 return ctx; 651 } 652 653 int amdgpu_ctx_put(struct amdgpu_ctx *ctx) 654 { 655 if (ctx == NULL) 656 return -EINVAL; 657 658 kref_put(&ctx->refcount, amdgpu_ctx_do_release); 659 return 0; 660 } 661 662 void amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, 663 struct drm_sched_entity *entity, 664 struct dma_fence *fence, uint64_t *handle) 665 { 666 struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); 667 uint64_t seq = centity->sequence; 668 struct dma_fence *other = NULL; 669 unsigned idx = 0; 670 671 idx = seq & (amdgpu_sched_jobs - 1); 672 other = centity->fences[idx]; 673 if (other) 674 BUG_ON(!dma_fence_is_signaled(other)); 675 676 dma_fence_get(fence); 677 678 spin_lock(&ctx->ring_lock); 679 centity->fences[idx] = fence; 680 centity->sequence++; 681 spin_unlock(&ctx->ring_lock); 682 683 dma_fence_put(other); 684 if (handle) 685 *handle = seq; 686 } 687 688 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, 689 struct drm_sched_entity *entity, 690 uint64_t seq) 691 { 692 struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); 693 struct dma_fence *fence; 694 695 spin_lock(&ctx->ring_lock); 696 697 if (seq == ~0ull) 698 seq = centity->sequence - 1; 699 700 if (seq >= centity->sequence) { 701 spin_unlock(&ctx->ring_lock); 702 return ERR_PTR(-EINVAL); 703 } 704 705 706 if (seq + amdgpu_sched_jobs < centity->sequence) { 707 spin_unlock(&ctx->ring_lock); 708 return NULL; 709 } 710 711 fence = dma_fence_get(centity->fences[seq & (amdgpu_sched_jobs - 1)]); 712 spin_unlock(&ctx->ring_lock); 713 714 return fence; 715 } 716 717 static void amdgpu_ctx_set_entity_priority(struct amdgpu_ctx *ctx, 718 struct amdgpu_ctx_entity *aentity, 719 int hw_ip, 720 int32_t priority) 721 { 722 struct amdgpu_device *adev = ctx->adev; 723 unsigned int hw_prio; 724 struct drm_gpu_scheduler **scheds = NULL; 725 unsigned num_scheds; 726 727 /* set sw priority */ 728 drm_sched_entity_set_priority(&aentity->entity, 729 amdgpu_ctx_to_drm_sched_prio(priority)); 730 731 /* set hw priority */ 732 if (hw_ip == AMDGPU_HW_IP_COMPUTE) { 733 hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip); 734 hw_prio = array_index_nospec(hw_prio, AMDGPU_RING_PRIO_MAX); 735 scheds = adev->gpu_sched[hw_ip][hw_prio].sched; 736 num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds; 737 drm_sched_entity_modify_sched(&aentity->entity, scheds, 738 num_scheds); 739 } 740 } 741 742 void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx, 743 int32_t priority) 744 { 745 int32_t ctx_prio; 746 unsigned i, j; 747 748 ctx->override_priority = priority; 749 750 ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ? 751 ctx->init_priority : ctx->override_priority; 752 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 753 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { 754 if (!ctx->entities[i][j]) 755 continue; 756 757 amdgpu_ctx_set_entity_priority(ctx, ctx->entities[i][j], 758 i, ctx_prio); 759 } 760 } 761 } 762 763 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, 764 struct drm_sched_entity *entity) 765 { 766 struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); 767 struct dma_fence *other; 768 unsigned idx; 769 long r; 770 771 spin_lock(&ctx->ring_lock); 772 idx = centity->sequence & (amdgpu_sched_jobs - 1); 773 other = dma_fence_get(centity->fences[idx]); 774 spin_unlock(&ctx->ring_lock); 775 776 if (!other) 777 return 0; 778 779 r = dma_fence_wait(other, true); 780 if (r < 0 && r != -ERESTARTSYS) 781 DRM_ERROR("Error (%ld) waiting for fence!\n", r); 782 783 dma_fence_put(other); 784 return r; 785 } 786 787 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr) 788 { 789 mutex_init(&mgr->lock); 790 idr_init(&mgr->ctx_handles); 791 } 792 793 long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout) 794 { 795 struct amdgpu_ctx *ctx; 796 struct idr *idp; 797 uint32_t id, i, j; 798 799 idp = &mgr->ctx_handles; 800 801 mutex_lock(&mgr->lock); 802 idr_for_each_entry(idp, ctx, id) { 803 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 804 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { 805 struct drm_sched_entity *entity; 806 807 if (!ctx->entities[i][j]) 808 continue; 809 810 entity = &ctx->entities[i][j]->entity; 811 timeout = drm_sched_entity_flush(entity, timeout); 812 } 813 } 814 } 815 mutex_unlock(&mgr->lock); 816 return timeout; 817 } 818 819 void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr) 820 { 821 struct amdgpu_ctx *ctx; 822 struct idr *idp; 823 uint32_t id, i, j; 824 825 idp = &mgr->ctx_handles; 826 827 idr_for_each_entry(idp, ctx, id) { 828 if (kref_read(&ctx->refcount) != 1) { 829 DRM_ERROR("ctx %p is still alive\n", ctx); 830 continue; 831 } 832 833 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 834 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { 835 struct drm_sched_entity *entity; 836 837 if (!ctx->entities[i][j]) 838 continue; 839 840 entity = &ctx->entities[i][j]->entity; 841 drm_sched_entity_fini(entity); 842 } 843 } 844 } 845 } 846 847 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr) 848 { 849 struct amdgpu_ctx *ctx; 850 struct idr *idp; 851 uint32_t id; 852 853 amdgpu_ctx_mgr_entity_fini(mgr); 854 855 idp = &mgr->ctx_handles; 856 857 idr_for_each_entry(idp, ctx, id) { 858 if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1) 859 DRM_ERROR("ctx %p is still alive\n", ctx); 860 } 861 862 idr_destroy(&mgr->ctx_handles); 863 mutex_destroy(&mgr->lock); 864 } 865 866 static void amdgpu_ctx_fence_time(struct amdgpu_ctx *ctx, 867 struct amdgpu_ctx_entity *centity, ktime_t *total, ktime_t *max) 868 { 869 ktime_t now, t1; 870 uint32_t i; 871 872 *total = *max = 0; 873 874 now = ktime_get(); 875 for (i = 0; i < amdgpu_sched_jobs; i++) { 876 struct dma_fence *fence; 877 struct drm_sched_fence *s_fence; 878 879 spin_lock(&ctx->ring_lock); 880 fence = dma_fence_get(centity->fences[i]); 881 spin_unlock(&ctx->ring_lock); 882 if (!fence) 883 continue; 884 s_fence = to_drm_sched_fence(fence); 885 if (!dma_fence_is_signaled(&s_fence->scheduled)) { 886 dma_fence_put(fence); 887 continue; 888 } 889 t1 = s_fence->scheduled.timestamp; 890 if (!ktime_before(t1, now)) { 891 dma_fence_put(fence); 892 continue; 893 } 894 if (dma_fence_is_signaled(&s_fence->finished) && 895 s_fence->finished.timestamp < now) 896 *total += ktime_sub(s_fence->finished.timestamp, t1); 897 else 898 *total += ktime_sub(now, t1); 899 t1 = ktime_sub(now, t1); 900 dma_fence_put(fence); 901 *max = max(t1, *max); 902 } 903 } 904 905 ktime_t amdgpu_ctx_mgr_fence_usage(struct amdgpu_ctx_mgr *mgr, uint32_t hwip, 906 uint32_t idx, uint64_t *elapsed) 907 { 908 struct idr *idp; 909 struct amdgpu_ctx *ctx; 910 uint32_t id; 911 struct amdgpu_ctx_entity *centity; 912 ktime_t total = 0, max = 0; 913 914 if (idx >= AMDGPU_MAX_ENTITY_NUM) 915 return 0; 916 idp = &mgr->ctx_handles; 917 mutex_lock(&mgr->lock); 918 idr_for_each_entry(idp, ctx, id) { 919 ktime_t ttotal, tmax; 920 921 if (!ctx->entities[hwip][idx]) 922 continue; 923 924 centity = ctx->entities[hwip][idx]; 925 amdgpu_ctx_fence_time(ctx, centity, &ttotal, &tmax); 926 927 /* Harmonic mean approximation diverges for very small 928 * values. If ratio < 0.01% ignore 929 */ 930 if (AMDGPU_CTX_FENCE_USAGE_MIN_RATIO(tmax, ttotal)) 931 continue; 932 933 total = ktime_add(total, ttotal); 934 max = ktime_after(tmax, max) ? tmax : max; 935 } 936 937 mutex_unlock(&mgr->lock); 938 if (elapsed) 939 *elapsed = max; 940 941 return total; 942 } 943