1 /* 2 * Copyright 2015 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: monk liu <monk.liu@amd.com> 23 */ 24 25 #include <drm/drm_auth.h> 26 #include <drm/drm_drv.h> 27 #include "amdgpu.h" 28 #include "amdgpu_sched.h" 29 #include "amdgpu_ras.h" 30 #include <linux/nospec.h> 31 32 #define to_amdgpu_ctx_entity(e) \ 33 container_of((e), struct amdgpu_ctx_entity, entity) 34 35 const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM] = { 36 [AMDGPU_HW_IP_GFX] = 1, 37 [AMDGPU_HW_IP_COMPUTE] = 4, 38 [AMDGPU_HW_IP_DMA] = 2, 39 [AMDGPU_HW_IP_UVD] = 1, 40 [AMDGPU_HW_IP_VCE] = 1, 41 [AMDGPU_HW_IP_UVD_ENC] = 1, 42 [AMDGPU_HW_IP_VCN_DEC] = 1, 43 [AMDGPU_HW_IP_VCN_ENC] = 1, 44 [AMDGPU_HW_IP_VCN_JPEG] = 1, 45 }; 46 47 bool amdgpu_ctx_priority_is_valid(int32_t ctx_prio) 48 { 49 switch (ctx_prio) { 50 case AMDGPU_CTX_PRIORITY_UNSET: 51 case AMDGPU_CTX_PRIORITY_VERY_LOW: 52 case AMDGPU_CTX_PRIORITY_LOW: 53 case AMDGPU_CTX_PRIORITY_NORMAL: 54 case AMDGPU_CTX_PRIORITY_HIGH: 55 case AMDGPU_CTX_PRIORITY_VERY_HIGH: 56 return true; 57 default: 58 return false; 59 } 60 } 61 62 static enum drm_sched_priority 63 amdgpu_ctx_to_drm_sched_prio(int32_t ctx_prio) 64 { 65 switch (ctx_prio) { 66 case AMDGPU_CTX_PRIORITY_UNSET: 67 return DRM_SCHED_PRIORITY_UNSET; 68 69 case AMDGPU_CTX_PRIORITY_VERY_LOW: 70 return DRM_SCHED_PRIORITY_MIN; 71 72 case AMDGPU_CTX_PRIORITY_LOW: 73 return DRM_SCHED_PRIORITY_MIN; 74 75 case AMDGPU_CTX_PRIORITY_NORMAL: 76 return DRM_SCHED_PRIORITY_NORMAL; 77 78 case AMDGPU_CTX_PRIORITY_HIGH: 79 return DRM_SCHED_PRIORITY_HIGH; 80 81 case AMDGPU_CTX_PRIORITY_VERY_HIGH: 82 return DRM_SCHED_PRIORITY_HIGH; 83 84 /* This should not happen as we sanitized userspace provided priority 85 * already, WARN if this happens. 86 */ 87 default: 88 WARN(1, "Invalid context priority %d\n", ctx_prio); 89 return DRM_SCHED_PRIORITY_NORMAL; 90 } 91 92 } 93 94 static int amdgpu_ctx_priority_permit(struct drm_file *filp, 95 int32_t priority) 96 { 97 if (!amdgpu_ctx_priority_is_valid(priority)) 98 return -EINVAL; 99 100 /* NORMAL and below are accessible by everyone */ 101 if (priority <= AMDGPU_CTX_PRIORITY_NORMAL) 102 return 0; 103 104 if (capable(CAP_SYS_NICE)) 105 return 0; 106 107 if (drm_is_current_master(filp)) 108 return 0; 109 110 return -EACCES; 111 } 112 113 static enum amdgpu_gfx_pipe_priority amdgpu_ctx_prio_to_gfx_pipe_prio(int32_t prio) 114 { 115 switch (prio) { 116 case AMDGPU_CTX_PRIORITY_HIGH: 117 case AMDGPU_CTX_PRIORITY_VERY_HIGH: 118 return AMDGPU_GFX_PIPE_PRIO_HIGH; 119 default: 120 return AMDGPU_GFX_PIPE_PRIO_NORMAL; 121 } 122 } 123 124 static enum amdgpu_ring_priority_level amdgpu_ctx_sched_prio_to_ring_prio(int32_t prio) 125 { 126 switch (prio) { 127 case AMDGPU_CTX_PRIORITY_HIGH: 128 return AMDGPU_RING_PRIO_1; 129 case AMDGPU_CTX_PRIORITY_VERY_HIGH: 130 return AMDGPU_RING_PRIO_2; 131 default: 132 return AMDGPU_RING_PRIO_0; 133 } 134 } 135 136 static unsigned int amdgpu_ctx_get_hw_prio(struct amdgpu_ctx *ctx, u32 hw_ip) 137 { 138 struct amdgpu_device *adev = ctx->mgr->adev; 139 unsigned int hw_prio; 140 int32_t ctx_prio; 141 142 ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ? 143 ctx->init_priority : ctx->override_priority; 144 145 switch (hw_ip) { 146 case AMDGPU_HW_IP_GFX: 147 case AMDGPU_HW_IP_COMPUTE: 148 hw_prio = amdgpu_ctx_prio_to_gfx_pipe_prio(ctx_prio); 149 break; 150 case AMDGPU_HW_IP_VCE: 151 case AMDGPU_HW_IP_VCN_ENC: 152 hw_prio = amdgpu_ctx_sched_prio_to_ring_prio(ctx_prio); 153 break; 154 default: 155 hw_prio = AMDGPU_RING_PRIO_DEFAULT; 156 break; 157 } 158 159 hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM); 160 if (adev->gpu_sched[hw_ip][hw_prio].num_scheds == 0) 161 hw_prio = AMDGPU_RING_PRIO_DEFAULT; 162 163 return hw_prio; 164 } 165 166 /* Calculate the time spend on the hw */ 167 static ktime_t amdgpu_ctx_fence_time(struct dma_fence *fence) 168 { 169 struct drm_sched_fence *s_fence; 170 171 if (!fence) 172 return ns_to_ktime(0); 173 174 /* When the fence is not even scheduled it can't have spend time */ 175 s_fence = to_drm_sched_fence(fence); 176 if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &s_fence->scheduled.flags)) 177 return ns_to_ktime(0); 178 179 /* When it is still running account how much already spend */ 180 if (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &s_fence->finished.flags)) 181 return ktime_sub(ktime_get(), s_fence->scheduled.timestamp); 182 183 return ktime_sub(s_fence->finished.timestamp, 184 s_fence->scheduled.timestamp); 185 } 186 187 static ktime_t amdgpu_ctx_entity_time(struct amdgpu_ctx *ctx, 188 struct amdgpu_ctx_entity *centity) 189 { 190 ktime_t res = ns_to_ktime(0); 191 uint32_t i; 192 193 spin_lock(&ctx->ring_lock); 194 for (i = 0; i < amdgpu_sched_jobs; i++) { 195 res = ktime_add(res, amdgpu_ctx_fence_time(centity->fences[i])); 196 } 197 spin_unlock(&ctx->ring_lock); 198 return res; 199 } 200 201 static int amdgpu_ctx_init_entity(struct amdgpu_ctx *ctx, u32 hw_ip, 202 const u32 ring) 203 { 204 struct drm_gpu_scheduler **scheds = NULL, *sched = NULL; 205 struct amdgpu_device *adev = ctx->mgr->adev; 206 struct amdgpu_ctx_entity *entity; 207 enum drm_sched_priority drm_prio; 208 unsigned int hw_prio, num_scheds; 209 int32_t ctx_prio; 210 int r; 211 212 entity = kzalloc(struct_size(entity, fences, amdgpu_sched_jobs), 213 GFP_KERNEL); 214 if (!entity) 215 return -ENOMEM; 216 217 ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ? 218 ctx->init_priority : ctx->override_priority; 219 entity->hw_ip = hw_ip; 220 entity->sequence = 1; 221 hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip); 222 drm_prio = amdgpu_ctx_to_drm_sched_prio(ctx_prio); 223 224 hw_ip = array_index_nospec(hw_ip, AMDGPU_HW_IP_NUM); 225 scheds = adev->gpu_sched[hw_ip][hw_prio].sched; 226 num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds; 227 228 /* disable load balance if the hw engine retains context among dependent jobs */ 229 if (hw_ip == AMDGPU_HW_IP_VCN_ENC || 230 hw_ip == AMDGPU_HW_IP_VCN_DEC || 231 hw_ip == AMDGPU_HW_IP_UVD_ENC || 232 hw_ip == AMDGPU_HW_IP_UVD) { 233 sched = drm_sched_pick_best(scheds, num_scheds); 234 scheds = &sched; 235 num_scheds = 1; 236 } 237 238 r = drm_sched_entity_init(&entity->entity, drm_prio, scheds, num_scheds, 239 &ctx->guilty); 240 if (r) 241 goto error_free_entity; 242 243 /* It's not an error if we fail to install the new entity */ 244 if (cmpxchg(&ctx->entities[hw_ip][ring], NULL, entity)) 245 goto cleanup_entity; 246 247 return 0; 248 249 cleanup_entity: 250 drm_sched_entity_fini(&entity->entity); 251 252 error_free_entity: 253 kfree(entity); 254 255 return r; 256 } 257 258 static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_ctx_entity *entity) 259 { 260 ktime_t res = ns_to_ktime(0); 261 int i; 262 263 if (!entity) 264 return res; 265 266 for (i = 0; i < amdgpu_sched_jobs; ++i) { 267 res = ktime_add(res, amdgpu_ctx_fence_time(entity->fences[i])); 268 dma_fence_put(entity->fences[i]); 269 } 270 271 kfree(entity); 272 return res; 273 } 274 275 static int amdgpu_ctx_get_stable_pstate(struct amdgpu_ctx *ctx, 276 u32 *stable_pstate) 277 { 278 struct amdgpu_device *adev = ctx->mgr->adev; 279 enum amd_dpm_forced_level current_level; 280 281 current_level = amdgpu_dpm_get_performance_level(adev); 282 283 switch (current_level) { 284 case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD: 285 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_STANDARD; 286 break; 287 case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK: 288 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK; 289 break; 290 case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK: 291 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK; 292 break; 293 case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK: 294 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_PEAK; 295 break; 296 default: 297 *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE; 298 break; 299 } 300 return 0; 301 } 302 303 static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority, 304 struct drm_file *filp, struct amdgpu_ctx *ctx) 305 { 306 struct amdgpu_fpriv *fpriv = filp->driver_priv; 307 u32 current_stable_pstate; 308 int r; 309 310 r = amdgpu_ctx_priority_permit(filp, priority); 311 if (r) 312 return r; 313 314 memset(ctx, 0, sizeof(*ctx)); 315 316 kref_init(&ctx->refcount); 317 ctx->mgr = mgr; 318 spin_lock_init(&ctx->ring_lock); 319 320 ctx->reset_counter = atomic_read(&mgr->adev->gpu_reset_counter); 321 ctx->reset_counter_query = ctx->reset_counter; 322 ctx->vram_lost_counter = atomic_read(&mgr->adev->vram_lost_counter); 323 ctx->init_priority = priority; 324 ctx->override_priority = AMDGPU_CTX_PRIORITY_UNSET; 325 326 r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate); 327 if (r) 328 return r; 329 330 if (mgr->adev->pm.stable_pstate_ctx) 331 ctx->stable_pstate = mgr->adev->pm.stable_pstate_ctx->stable_pstate; 332 else 333 ctx->stable_pstate = current_stable_pstate; 334 335 ctx->ctx_mgr = &(fpriv->ctx_mgr); 336 return 0; 337 } 338 339 static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, 340 u32 stable_pstate) 341 { 342 struct amdgpu_device *adev = ctx->mgr->adev; 343 enum amd_dpm_forced_level level; 344 u32 current_stable_pstate; 345 int r; 346 347 mutex_lock(&adev->pm.stable_pstate_ctx_lock); 348 if (adev->pm.stable_pstate_ctx && adev->pm.stable_pstate_ctx != ctx) { 349 r = -EBUSY; 350 goto done; 351 } 352 353 r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate); 354 if (r || (stable_pstate == current_stable_pstate)) 355 goto done; 356 357 switch (stable_pstate) { 358 case AMDGPU_CTX_STABLE_PSTATE_NONE: 359 level = AMD_DPM_FORCED_LEVEL_AUTO; 360 break; 361 case AMDGPU_CTX_STABLE_PSTATE_STANDARD: 362 level = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD; 363 break; 364 case AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK: 365 level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK; 366 break; 367 case AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK: 368 level = AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK; 369 break; 370 case AMDGPU_CTX_STABLE_PSTATE_PEAK: 371 level = AMD_DPM_FORCED_LEVEL_PROFILE_PEAK; 372 break; 373 default: 374 r = -EINVAL; 375 goto done; 376 } 377 378 r = amdgpu_dpm_force_performance_level(adev, level); 379 380 if (level == AMD_DPM_FORCED_LEVEL_AUTO) 381 adev->pm.stable_pstate_ctx = NULL; 382 else 383 adev->pm.stable_pstate_ctx = ctx; 384 done: 385 mutex_unlock(&adev->pm.stable_pstate_ctx_lock); 386 387 return r; 388 } 389 390 static void amdgpu_ctx_fini(struct kref *ref) 391 { 392 struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount); 393 struct amdgpu_ctx_mgr *mgr = ctx->mgr; 394 struct amdgpu_device *adev = mgr->adev; 395 unsigned i, j, idx; 396 397 if (!adev) 398 return; 399 400 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 401 for (j = 0; j < AMDGPU_MAX_ENTITY_NUM; ++j) { 402 ktime_t spend; 403 404 spend = amdgpu_ctx_fini_entity(ctx->entities[i][j]); 405 atomic64_add(ktime_to_ns(spend), &mgr->time_spend[i]); 406 } 407 } 408 409 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 410 amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate); 411 drm_dev_exit(idx); 412 } 413 414 kfree(ctx); 415 } 416 417 int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance, 418 u32 ring, struct drm_sched_entity **entity) 419 { 420 int r; 421 422 if (hw_ip >= AMDGPU_HW_IP_NUM) { 423 DRM_ERROR("unknown HW IP type: %d\n", hw_ip); 424 return -EINVAL; 425 } 426 427 /* Right now all IPs have only one instance - multiple rings. */ 428 if (instance != 0) { 429 DRM_DEBUG("invalid ip instance: %d\n", instance); 430 return -EINVAL; 431 } 432 433 if (ring >= amdgpu_ctx_num_entities[hw_ip]) { 434 DRM_DEBUG("invalid ring: %d %d\n", hw_ip, ring); 435 return -EINVAL; 436 } 437 438 if (ctx->entities[hw_ip][ring] == NULL) { 439 r = amdgpu_ctx_init_entity(ctx, hw_ip, ring); 440 if (r) 441 return r; 442 } 443 444 *entity = &ctx->entities[hw_ip][ring]->entity; 445 return 0; 446 } 447 448 static int amdgpu_ctx_alloc(struct amdgpu_device *adev, 449 struct amdgpu_fpriv *fpriv, 450 struct drm_file *filp, 451 int32_t priority, 452 uint32_t *id) 453 { 454 struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; 455 struct amdgpu_ctx *ctx; 456 int r; 457 458 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 459 if (!ctx) 460 return -ENOMEM; 461 462 mutex_lock(&mgr->lock); 463 r = idr_alloc(&mgr->ctx_handles, ctx, 1, AMDGPU_VM_MAX_NUM_CTX, GFP_KERNEL); 464 if (r < 0) { 465 mutex_unlock(&mgr->lock); 466 kfree(ctx); 467 return r; 468 } 469 470 *id = (uint32_t)r; 471 r = amdgpu_ctx_init(mgr, priority, filp, ctx); 472 if (r) { 473 idr_remove(&mgr->ctx_handles, *id); 474 *id = 0; 475 kfree(ctx); 476 } 477 mutex_unlock(&mgr->lock); 478 return r; 479 } 480 481 static void amdgpu_ctx_do_release(struct kref *ref) 482 { 483 struct amdgpu_ctx *ctx; 484 u32 i, j; 485 486 ctx = container_of(ref, struct amdgpu_ctx, refcount); 487 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 488 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { 489 if (!ctx->entities[i][j]) 490 continue; 491 492 drm_sched_entity_destroy(&ctx->entities[i][j]->entity); 493 } 494 } 495 496 amdgpu_ctx_fini(ref); 497 } 498 499 static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id) 500 { 501 struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; 502 struct amdgpu_ctx *ctx; 503 504 mutex_lock(&mgr->lock); 505 ctx = idr_remove(&mgr->ctx_handles, id); 506 if (ctx) 507 kref_put(&ctx->refcount, amdgpu_ctx_do_release); 508 mutex_unlock(&mgr->lock); 509 return ctx ? 0 : -EINVAL; 510 } 511 512 static int amdgpu_ctx_query(struct amdgpu_device *adev, 513 struct amdgpu_fpriv *fpriv, uint32_t id, 514 union drm_amdgpu_ctx_out *out) 515 { 516 struct amdgpu_ctx *ctx; 517 struct amdgpu_ctx_mgr *mgr; 518 unsigned reset_counter; 519 520 if (!fpriv) 521 return -EINVAL; 522 523 mgr = &fpriv->ctx_mgr; 524 mutex_lock(&mgr->lock); 525 ctx = idr_find(&mgr->ctx_handles, id); 526 if (!ctx) { 527 mutex_unlock(&mgr->lock); 528 return -EINVAL; 529 } 530 531 /* TODO: these two are always zero */ 532 out->state.flags = 0x0; 533 out->state.hangs = 0x0; 534 535 /* determine if a GPU reset has occured since the last call */ 536 reset_counter = atomic_read(&adev->gpu_reset_counter); 537 /* TODO: this should ideally return NO, GUILTY, or INNOCENT. */ 538 if (ctx->reset_counter_query == reset_counter) 539 out->state.reset_status = AMDGPU_CTX_NO_RESET; 540 else 541 out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET; 542 ctx->reset_counter_query = reset_counter; 543 544 mutex_unlock(&mgr->lock); 545 return 0; 546 } 547 548 #define AMDGPU_RAS_COUNTE_DELAY_MS 3000 549 550 static int amdgpu_ctx_query2(struct amdgpu_device *adev, 551 struct amdgpu_fpriv *fpriv, uint32_t id, 552 union drm_amdgpu_ctx_out *out) 553 { 554 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 555 struct amdgpu_ctx *ctx; 556 struct amdgpu_ctx_mgr *mgr; 557 558 if (!fpriv) 559 return -EINVAL; 560 561 mgr = &fpriv->ctx_mgr; 562 mutex_lock(&mgr->lock); 563 ctx = idr_find(&mgr->ctx_handles, id); 564 if (!ctx) { 565 mutex_unlock(&mgr->lock); 566 return -EINVAL; 567 } 568 569 out->state.flags = 0x0; 570 out->state.hangs = 0x0; 571 572 if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter)) 573 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET; 574 575 if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) 576 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST; 577 578 if (atomic_read(&ctx->guilty)) 579 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY; 580 581 if (amdgpu_in_reset(adev)) 582 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET_IN_PROGRESS; 583 584 if (adev->ras_enabled && con) { 585 /* Return the cached values in O(1), 586 * and schedule delayed work to cache 587 * new vaues. 588 */ 589 int ce_count, ue_count; 590 591 ce_count = atomic_read(&con->ras_ce_count); 592 ue_count = atomic_read(&con->ras_ue_count); 593 594 if (ce_count != ctx->ras_counter_ce) { 595 ctx->ras_counter_ce = ce_count; 596 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_CE; 597 } 598 599 if (ue_count != ctx->ras_counter_ue) { 600 ctx->ras_counter_ue = ue_count; 601 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_UE; 602 } 603 604 schedule_delayed_work(&con->ras_counte_delay_work, 605 msecs_to_jiffies(AMDGPU_RAS_COUNTE_DELAY_MS)); 606 } 607 608 mutex_unlock(&mgr->lock); 609 return 0; 610 } 611 612 613 614 static int amdgpu_ctx_stable_pstate(struct amdgpu_device *adev, 615 struct amdgpu_fpriv *fpriv, uint32_t id, 616 bool set, u32 *stable_pstate) 617 { 618 struct amdgpu_ctx *ctx; 619 struct amdgpu_ctx_mgr *mgr; 620 int r; 621 622 if (!fpriv) 623 return -EINVAL; 624 625 mgr = &fpriv->ctx_mgr; 626 mutex_lock(&mgr->lock); 627 ctx = idr_find(&mgr->ctx_handles, id); 628 if (!ctx) { 629 mutex_unlock(&mgr->lock); 630 return -EINVAL; 631 } 632 633 if (set) 634 r = amdgpu_ctx_set_stable_pstate(ctx, *stable_pstate); 635 else 636 r = amdgpu_ctx_get_stable_pstate(ctx, stable_pstate); 637 638 mutex_unlock(&mgr->lock); 639 return r; 640 } 641 642 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, 643 struct drm_file *filp) 644 { 645 int r; 646 uint32_t id, stable_pstate; 647 int32_t priority; 648 649 union drm_amdgpu_ctx *args = data; 650 struct amdgpu_device *adev = drm_to_adev(dev); 651 struct amdgpu_fpriv *fpriv = filp->driver_priv; 652 653 id = args->in.ctx_id; 654 priority = args->in.priority; 655 656 /* For backwards compatibility reasons, we need to accept 657 * ioctls with garbage in the priority field */ 658 if (!amdgpu_ctx_priority_is_valid(priority)) 659 priority = AMDGPU_CTX_PRIORITY_NORMAL; 660 661 switch (args->in.op) { 662 case AMDGPU_CTX_OP_ALLOC_CTX: 663 r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id); 664 args->out.alloc.ctx_id = id; 665 break; 666 case AMDGPU_CTX_OP_FREE_CTX: 667 r = amdgpu_ctx_free(fpriv, id); 668 break; 669 case AMDGPU_CTX_OP_QUERY_STATE: 670 r = amdgpu_ctx_query(adev, fpriv, id, &args->out); 671 break; 672 case AMDGPU_CTX_OP_QUERY_STATE2: 673 r = amdgpu_ctx_query2(adev, fpriv, id, &args->out); 674 break; 675 case AMDGPU_CTX_OP_GET_STABLE_PSTATE: 676 if (args->in.flags) 677 return -EINVAL; 678 r = amdgpu_ctx_stable_pstate(adev, fpriv, id, false, &stable_pstate); 679 if (!r) 680 args->out.pstate.flags = stable_pstate; 681 break; 682 case AMDGPU_CTX_OP_SET_STABLE_PSTATE: 683 if (args->in.flags & ~AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK) 684 return -EINVAL; 685 stable_pstate = args->in.flags & AMDGPU_CTX_STABLE_PSTATE_FLAGS_MASK; 686 if (stable_pstate > AMDGPU_CTX_STABLE_PSTATE_PEAK) 687 return -EINVAL; 688 r = amdgpu_ctx_stable_pstate(adev, fpriv, id, true, &stable_pstate); 689 break; 690 default: 691 return -EINVAL; 692 } 693 694 return r; 695 } 696 697 struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id) 698 { 699 struct amdgpu_ctx *ctx; 700 struct amdgpu_ctx_mgr *mgr; 701 702 if (!fpriv) 703 return NULL; 704 705 mgr = &fpriv->ctx_mgr; 706 707 mutex_lock(&mgr->lock); 708 ctx = idr_find(&mgr->ctx_handles, id); 709 if (ctx) 710 kref_get(&ctx->refcount); 711 mutex_unlock(&mgr->lock); 712 return ctx; 713 } 714 715 int amdgpu_ctx_put(struct amdgpu_ctx *ctx) 716 { 717 if (ctx == NULL) 718 return -EINVAL; 719 720 kref_put(&ctx->refcount, amdgpu_ctx_do_release); 721 return 0; 722 } 723 724 uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, 725 struct drm_sched_entity *entity, 726 struct dma_fence *fence) 727 { 728 struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); 729 uint64_t seq = centity->sequence; 730 struct dma_fence *other = NULL; 731 unsigned idx = 0; 732 733 idx = seq & (amdgpu_sched_jobs - 1); 734 other = centity->fences[idx]; 735 WARN_ON(other && !dma_fence_is_signaled(other)); 736 737 dma_fence_get(fence); 738 739 spin_lock(&ctx->ring_lock); 740 centity->fences[idx] = fence; 741 centity->sequence++; 742 spin_unlock(&ctx->ring_lock); 743 744 atomic64_add(ktime_to_ns(amdgpu_ctx_fence_time(other)), 745 &ctx->mgr->time_spend[centity->hw_ip]); 746 747 dma_fence_put(other); 748 return seq; 749 } 750 751 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, 752 struct drm_sched_entity *entity, 753 uint64_t seq) 754 { 755 struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); 756 struct dma_fence *fence; 757 758 spin_lock(&ctx->ring_lock); 759 760 if (seq == ~0ull) 761 seq = centity->sequence - 1; 762 763 if (seq >= centity->sequence) { 764 spin_unlock(&ctx->ring_lock); 765 return ERR_PTR(-EINVAL); 766 } 767 768 769 if (seq + amdgpu_sched_jobs < centity->sequence) { 770 spin_unlock(&ctx->ring_lock); 771 return NULL; 772 } 773 774 fence = dma_fence_get(centity->fences[seq & (amdgpu_sched_jobs - 1)]); 775 spin_unlock(&ctx->ring_lock); 776 777 return fence; 778 } 779 780 static void amdgpu_ctx_set_entity_priority(struct amdgpu_ctx *ctx, 781 struct amdgpu_ctx_entity *aentity, 782 int hw_ip, 783 int32_t priority) 784 { 785 struct amdgpu_device *adev = ctx->mgr->adev; 786 unsigned int hw_prio; 787 struct drm_gpu_scheduler **scheds = NULL; 788 unsigned num_scheds; 789 790 /* set sw priority */ 791 drm_sched_entity_set_priority(&aentity->entity, 792 amdgpu_ctx_to_drm_sched_prio(priority)); 793 794 /* set hw priority */ 795 if (hw_ip == AMDGPU_HW_IP_COMPUTE || hw_ip == AMDGPU_HW_IP_GFX) { 796 hw_prio = amdgpu_ctx_get_hw_prio(ctx, hw_ip); 797 hw_prio = array_index_nospec(hw_prio, AMDGPU_RING_PRIO_MAX); 798 scheds = adev->gpu_sched[hw_ip][hw_prio].sched; 799 num_scheds = adev->gpu_sched[hw_ip][hw_prio].num_scheds; 800 drm_sched_entity_modify_sched(&aentity->entity, scheds, 801 num_scheds); 802 } 803 } 804 805 void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx, 806 int32_t priority) 807 { 808 int32_t ctx_prio; 809 unsigned i, j; 810 811 ctx->override_priority = priority; 812 813 ctx_prio = (ctx->override_priority == AMDGPU_CTX_PRIORITY_UNSET) ? 814 ctx->init_priority : ctx->override_priority; 815 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 816 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { 817 if (!ctx->entities[i][j]) 818 continue; 819 820 amdgpu_ctx_set_entity_priority(ctx, ctx->entities[i][j], 821 i, ctx_prio); 822 } 823 } 824 } 825 826 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, 827 struct drm_sched_entity *entity) 828 { 829 struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity); 830 struct dma_fence *other; 831 unsigned idx; 832 long r; 833 834 spin_lock(&ctx->ring_lock); 835 idx = centity->sequence & (amdgpu_sched_jobs - 1); 836 other = dma_fence_get(centity->fences[idx]); 837 spin_unlock(&ctx->ring_lock); 838 839 if (!other) 840 return 0; 841 842 r = dma_fence_wait(other, true); 843 if (r < 0 && r != -ERESTARTSYS) 844 DRM_ERROR("Error (%ld) waiting for fence!\n", r); 845 846 dma_fence_put(other); 847 return r; 848 } 849 850 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr, 851 struct amdgpu_device *adev) 852 { 853 unsigned int i; 854 855 mgr->adev = adev; 856 mutex_init(&mgr->lock); 857 idr_init_base(&mgr->ctx_handles, 1); 858 859 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) 860 atomic64_set(&mgr->time_spend[i], 0); 861 } 862 863 long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout) 864 { 865 struct amdgpu_ctx *ctx; 866 struct idr *idp; 867 uint32_t id, i, j; 868 869 idp = &mgr->ctx_handles; 870 871 mutex_lock(&mgr->lock); 872 idr_for_each_entry(idp, ctx, id) { 873 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 874 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { 875 struct drm_sched_entity *entity; 876 877 if (!ctx->entities[i][j]) 878 continue; 879 880 entity = &ctx->entities[i][j]->entity; 881 timeout = drm_sched_entity_flush(entity, timeout); 882 } 883 } 884 } 885 mutex_unlock(&mgr->lock); 886 return timeout; 887 } 888 889 void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr) 890 { 891 struct amdgpu_ctx *ctx; 892 struct idr *idp; 893 uint32_t id, i, j; 894 895 idp = &mgr->ctx_handles; 896 897 idr_for_each_entry(idp, ctx, id) { 898 if (kref_read(&ctx->refcount) != 1) { 899 DRM_ERROR("ctx %p is still alive\n", ctx); 900 continue; 901 } 902 903 for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { 904 for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { 905 struct drm_sched_entity *entity; 906 907 if (!ctx->entities[i][j]) 908 continue; 909 910 entity = &ctx->entities[i][j]->entity; 911 drm_sched_entity_fini(entity); 912 } 913 } 914 } 915 } 916 917 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr) 918 { 919 struct amdgpu_ctx *ctx; 920 struct idr *idp; 921 uint32_t id; 922 923 amdgpu_ctx_mgr_entity_fini(mgr); 924 925 idp = &mgr->ctx_handles; 926 927 idr_for_each_entry(idp, ctx, id) { 928 if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1) 929 DRM_ERROR("ctx %p is still alive\n", ctx); 930 } 931 932 idr_destroy(&mgr->ctx_handles); 933 mutex_destroy(&mgr->lock); 934 } 935 936 void amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr *mgr, 937 ktime_t usage[AMDGPU_HW_IP_NUM]) 938 { 939 struct amdgpu_ctx *ctx; 940 unsigned int hw_ip, i; 941 uint32_t id; 942 943 /* 944 * This is a little bit racy because it can be that a ctx or a fence are 945 * destroyed just in the moment we try to account them. But that is ok 946 * since exactly that case is explicitely allowed by the interface. 947 */ 948 mutex_lock(&mgr->lock); 949 for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) { 950 uint64_t ns = atomic64_read(&mgr->time_spend[hw_ip]); 951 952 usage[hw_ip] = ns_to_ktime(ns); 953 } 954 955 idr_for_each_entry(&mgr->ctx_handles, ctx, id) { 956 for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) { 957 for (i = 0; i < amdgpu_ctx_num_entities[hw_ip]; ++i) { 958 struct amdgpu_ctx_entity *centity; 959 ktime_t spend; 960 961 centity = ctx->entities[hw_ip][i]; 962 if (!centity) 963 continue; 964 spend = amdgpu_ctx_entity_time(ctx, centity); 965 usage[hw_ip] = ktime_add(usage[hw_ip], spend); 966 } 967 } 968 } 969 mutex_unlock(&mgr->lock); 970 } 971