1 /* 2 * Copyright 2015 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: monk liu <monk.liu@amd.com> 23 */ 24 25 #include <drm/drmP.h> 26 #include <drm/drm_auth.h> 27 #include "amdgpu.h" 28 #include "amdgpu_sched.h" 29 30 static int amdgpu_ctx_priority_permit(struct drm_file *filp, 31 enum drm_sched_priority priority) 32 { 33 /* NORMAL and below are accessible by everyone */ 34 if (priority <= DRM_SCHED_PRIORITY_NORMAL) 35 return 0; 36 37 if (capable(CAP_SYS_NICE)) 38 return 0; 39 40 if (drm_is_current_master(filp)) 41 return 0; 42 43 return -EACCES; 44 } 45 46 static int amdgpu_ctx_init(struct amdgpu_device *adev, 47 enum drm_sched_priority priority, 48 struct drm_file *filp, 49 struct amdgpu_ctx *ctx) 50 { 51 unsigned i, j; 52 int r; 53 54 if (priority < 0 || priority >= DRM_SCHED_PRIORITY_MAX) 55 return -EINVAL; 56 57 r = amdgpu_ctx_priority_permit(filp, priority); 58 if (r) 59 return r; 60 61 memset(ctx, 0, sizeof(*ctx)); 62 ctx->adev = adev; 63 kref_init(&ctx->refcount); 64 spin_lock_init(&ctx->ring_lock); 65 ctx->fences = kcalloc(amdgpu_sched_jobs * AMDGPU_MAX_RINGS, 66 sizeof(struct dma_fence*), GFP_KERNEL); 67 if (!ctx->fences) 68 return -ENOMEM; 69 70 mutex_init(&ctx->lock); 71 72 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 73 ctx->rings[i].sequence = 1; 74 ctx->rings[i].fences = &ctx->fences[amdgpu_sched_jobs * i]; 75 } 76 77 ctx->reset_counter = atomic_read(&adev->gpu_reset_counter); 78 ctx->reset_counter_query = ctx->reset_counter; 79 ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter); 80 ctx->init_priority = priority; 81 ctx->override_priority = DRM_SCHED_PRIORITY_UNSET; 82 83 /* create context entity for each ring */ 84 for (i = 0; i < adev->num_rings; i++) { 85 struct amdgpu_ring *ring = adev->rings[i]; 86 struct drm_sched_rq *rq; 87 88 rq = &ring->sched.sched_rq[priority]; 89 90 if (ring == &adev->gfx.kiq.ring) 91 continue; 92 93 r = drm_sched_entity_init(&ring->sched, &ctx->rings[i].entity, 94 rq, &ctx->guilty); 95 if (r) 96 goto failed; 97 } 98 99 r = amdgpu_queue_mgr_init(adev, &ctx->queue_mgr); 100 if (r) 101 goto failed; 102 103 return 0; 104 105 failed: 106 for (j = 0; j < i; j++) 107 drm_sched_entity_fini(&adev->rings[j]->sched, 108 &ctx->rings[j].entity); 109 kfree(ctx->fences); 110 ctx->fences = NULL; 111 return r; 112 } 113 114 static void amdgpu_ctx_fini(struct kref *ref) 115 { 116 struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount); 117 struct amdgpu_device *adev = ctx->adev; 118 unsigned i, j; 119 120 if (!adev) 121 return; 122 123 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) 124 for (j = 0; j < amdgpu_sched_jobs; ++j) 125 dma_fence_put(ctx->rings[i].fences[j]); 126 kfree(ctx->fences); 127 ctx->fences = NULL; 128 129 amdgpu_queue_mgr_fini(adev, &ctx->queue_mgr); 130 131 mutex_destroy(&ctx->lock); 132 133 kfree(ctx); 134 } 135 136 static int amdgpu_ctx_alloc(struct amdgpu_device *adev, 137 struct amdgpu_fpriv *fpriv, 138 struct drm_file *filp, 139 enum drm_sched_priority priority, 140 uint32_t *id) 141 { 142 struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; 143 struct amdgpu_ctx *ctx; 144 int r; 145 146 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 147 if (!ctx) 148 return -ENOMEM; 149 150 mutex_lock(&mgr->lock); 151 r = idr_alloc(&mgr->ctx_handles, ctx, 1, 0, GFP_KERNEL); 152 if (r < 0) { 153 mutex_unlock(&mgr->lock); 154 kfree(ctx); 155 return r; 156 } 157 158 *id = (uint32_t)r; 159 r = amdgpu_ctx_init(adev, priority, filp, ctx); 160 if (r) { 161 idr_remove(&mgr->ctx_handles, *id); 162 *id = 0; 163 kfree(ctx); 164 } 165 mutex_unlock(&mgr->lock); 166 return r; 167 } 168 169 static void amdgpu_ctx_do_release(struct kref *ref) 170 { 171 struct amdgpu_ctx *ctx; 172 u32 i; 173 174 ctx = container_of(ref, struct amdgpu_ctx, refcount); 175 176 for (i = 0; i < ctx->adev->num_rings; i++) { 177 178 if (ctx->adev->rings[i] == &ctx->adev->gfx.kiq.ring) 179 continue; 180 181 drm_sched_entity_fini(&ctx->adev->rings[i]->sched, 182 &ctx->rings[i].entity); 183 } 184 185 amdgpu_ctx_fini(ref); 186 } 187 188 static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id) 189 { 190 struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; 191 struct amdgpu_ctx *ctx; 192 193 mutex_lock(&mgr->lock); 194 ctx = idr_remove(&mgr->ctx_handles, id); 195 if (ctx) 196 kref_put(&ctx->refcount, amdgpu_ctx_do_release); 197 mutex_unlock(&mgr->lock); 198 return ctx ? 0 : -EINVAL; 199 } 200 201 static int amdgpu_ctx_query(struct amdgpu_device *adev, 202 struct amdgpu_fpriv *fpriv, uint32_t id, 203 union drm_amdgpu_ctx_out *out) 204 { 205 struct amdgpu_ctx *ctx; 206 struct amdgpu_ctx_mgr *mgr; 207 unsigned reset_counter; 208 209 if (!fpriv) 210 return -EINVAL; 211 212 mgr = &fpriv->ctx_mgr; 213 mutex_lock(&mgr->lock); 214 ctx = idr_find(&mgr->ctx_handles, id); 215 if (!ctx) { 216 mutex_unlock(&mgr->lock); 217 return -EINVAL; 218 } 219 220 /* TODO: these two are always zero */ 221 out->state.flags = 0x0; 222 out->state.hangs = 0x0; 223 224 /* determine if a GPU reset has occured since the last call */ 225 reset_counter = atomic_read(&adev->gpu_reset_counter); 226 /* TODO: this should ideally return NO, GUILTY, or INNOCENT. */ 227 if (ctx->reset_counter_query == reset_counter) 228 out->state.reset_status = AMDGPU_CTX_NO_RESET; 229 else 230 out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET; 231 ctx->reset_counter_query = reset_counter; 232 233 mutex_unlock(&mgr->lock); 234 return 0; 235 } 236 237 static int amdgpu_ctx_query2(struct amdgpu_device *adev, 238 struct amdgpu_fpriv *fpriv, uint32_t id, 239 union drm_amdgpu_ctx_out *out) 240 { 241 struct amdgpu_ctx *ctx; 242 struct amdgpu_ctx_mgr *mgr; 243 244 if (!fpriv) 245 return -EINVAL; 246 247 mgr = &fpriv->ctx_mgr; 248 mutex_lock(&mgr->lock); 249 ctx = idr_find(&mgr->ctx_handles, id); 250 if (!ctx) { 251 mutex_unlock(&mgr->lock); 252 return -EINVAL; 253 } 254 255 out->state.flags = 0x0; 256 out->state.hangs = 0x0; 257 258 if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter)) 259 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET; 260 261 if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) 262 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST; 263 264 if (atomic_read(&ctx->guilty)) 265 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY; 266 267 mutex_unlock(&mgr->lock); 268 return 0; 269 } 270 271 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, 272 struct drm_file *filp) 273 { 274 int r; 275 uint32_t id; 276 enum drm_sched_priority priority; 277 278 union drm_amdgpu_ctx *args = data; 279 struct amdgpu_device *adev = dev->dev_private; 280 struct amdgpu_fpriv *fpriv = filp->driver_priv; 281 282 r = 0; 283 id = args->in.ctx_id; 284 priority = amdgpu_to_sched_priority(args->in.priority); 285 286 /* For backwards compatibility reasons, we need to accept 287 * ioctls with garbage in the priority field */ 288 if (priority == DRM_SCHED_PRIORITY_INVALID) 289 priority = DRM_SCHED_PRIORITY_NORMAL; 290 291 switch (args->in.op) { 292 case AMDGPU_CTX_OP_ALLOC_CTX: 293 r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id); 294 args->out.alloc.ctx_id = id; 295 break; 296 case AMDGPU_CTX_OP_FREE_CTX: 297 r = amdgpu_ctx_free(fpriv, id); 298 break; 299 case AMDGPU_CTX_OP_QUERY_STATE: 300 r = amdgpu_ctx_query(adev, fpriv, id, &args->out); 301 break; 302 case AMDGPU_CTX_OP_QUERY_STATE2: 303 r = amdgpu_ctx_query2(adev, fpriv, id, &args->out); 304 break; 305 default: 306 return -EINVAL; 307 } 308 309 return r; 310 } 311 312 struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id) 313 { 314 struct amdgpu_ctx *ctx; 315 struct amdgpu_ctx_mgr *mgr; 316 317 if (!fpriv) 318 return NULL; 319 320 mgr = &fpriv->ctx_mgr; 321 322 mutex_lock(&mgr->lock); 323 ctx = idr_find(&mgr->ctx_handles, id); 324 if (ctx) 325 kref_get(&ctx->refcount); 326 mutex_unlock(&mgr->lock); 327 return ctx; 328 } 329 330 int amdgpu_ctx_put(struct amdgpu_ctx *ctx) 331 { 332 if (ctx == NULL) 333 return -EINVAL; 334 335 kref_put(&ctx->refcount, amdgpu_ctx_do_release); 336 return 0; 337 } 338 339 int amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, 340 struct dma_fence *fence, uint64_t* handler) 341 { 342 struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx]; 343 uint64_t seq = cring->sequence; 344 unsigned idx = 0; 345 struct dma_fence *other = NULL; 346 347 idx = seq & (amdgpu_sched_jobs - 1); 348 other = cring->fences[idx]; 349 if (other) 350 BUG_ON(!dma_fence_is_signaled(other)); 351 352 dma_fence_get(fence); 353 354 spin_lock(&ctx->ring_lock); 355 cring->fences[idx] = fence; 356 cring->sequence++; 357 spin_unlock(&ctx->ring_lock); 358 359 dma_fence_put(other); 360 if (handler) 361 *handler = seq; 362 363 return 0; 364 } 365 366 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, 367 struct amdgpu_ring *ring, uint64_t seq) 368 { 369 struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx]; 370 struct dma_fence *fence; 371 372 spin_lock(&ctx->ring_lock); 373 374 if (seq == ~0ull) 375 seq = ctx->rings[ring->idx].sequence - 1; 376 377 if (seq >= cring->sequence) { 378 spin_unlock(&ctx->ring_lock); 379 return ERR_PTR(-EINVAL); 380 } 381 382 383 if (seq + amdgpu_sched_jobs < cring->sequence) { 384 spin_unlock(&ctx->ring_lock); 385 return NULL; 386 } 387 388 fence = dma_fence_get(cring->fences[seq & (amdgpu_sched_jobs - 1)]); 389 spin_unlock(&ctx->ring_lock); 390 391 return fence; 392 } 393 394 void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx, 395 enum drm_sched_priority priority) 396 { 397 int i; 398 struct amdgpu_device *adev = ctx->adev; 399 struct drm_sched_rq *rq; 400 struct drm_sched_entity *entity; 401 struct amdgpu_ring *ring; 402 enum drm_sched_priority ctx_prio; 403 404 ctx->override_priority = priority; 405 406 ctx_prio = (ctx->override_priority == DRM_SCHED_PRIORITY_UNSET) ? 407 ctx->init_priority : ctx->override_priority; 408 409 for (i = 0; i < adev->num_rings; i++) { 410 ring = adev->rings[i]; 411 entity = &ctx->rings[i].entity; 412 rq = &ring->sched.sched_rq[ctx_prio]; 413 414 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 415 continue; 416 417 drm_sched_entity_set_rq(entity, rq); 418 } 419 } 420 421 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, unsigned ring_id) 422 { 423 struct amdgpu_ctx_ring *cring = &ctx->rings[ring_id]; 424 unsigned idx = cring->sequence & (amdgpu_sched_jobs - 1); 425 struct dma_fence *other = cring->fences[idx]; 426 427 if (other) { 428 signed long r; 429 r = dma_fence_wait(other, true); 430 if (r < 0) { 431 if (r != -ERESTARTSYS) 432 DRM_ERROR("Error (%ld) waiting for fence!\n", r); 433 434 return r; 435 } 436 } 437 438 return 0; 439 } 440 441 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr) 442 { 443 mutex_init(&mgr->lock); 444 idr_init(&mgr->ctx_handles); 445 } 446 447 void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr) 448 { 449 struct amdgpu_ctx *ctx; 450 struct idr *idp; 451 uint32_t id, i; 452 453 idp = &mgr->ctx_handles; 454 455 idr_for_each_entry(idp, ctx, id) { 456 457 if (!ctx->adev) 458 return; 459 460 for (i = 0; i < ctx->adev->num_rings; i++) { 461 462 if (ctx->adev->rings[i] == &ctx->adev->gfx.kiq.ring) 463 continue; 464 465 if (kref_read(&ctx->refcount) == 1) 466 drm_sched_entity_do_release(&ctx->adev->rings[i]->sched, 467 &ctx->rings[i].entity); 468 else 469 DRM_ERROR("ctx %p is still alive\n", ctx); 470 } 471 } 472 } 473 474 void amdgpu_ctx_mgr_entity_cleanup(struct amdgpu_ctx_mgr *mgr) 475 { 476 struct amdgpu_ctx *ctx; 477 struct idr *idp; 478 uint32_t id, i; 479 480 idp = &mgr->ctx_handles; 481 482 idr_for_each_entry(idp, ctx, id) { 483 484 if (!ctx->adev) 485 return; 486 487 for (i = 0; i < ctx->adev->num_rings; i++) { 488 489 if (ctx->adev->rings[i] == &ctx->adev->gfx.kiq.ring) 490 continue; 491 492 if (kref_read(&ctx->refcount) == 1) 493 drm_sched_entity_cleanup(&ctx->adev->rings[i]->sched, 494 &ctx->rings[i].entity); 495 else 496 DRM_ERROR("ctx %p is still alive\n", ctx); 497 } 498 } 499 } 500 501 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr) 502 { 503 struct amdgpu_ctx *ctx; 504 struct idr *idp; 505 uint32_t id; 506 507 amdgpu_ctx_mgr_entity_cleanup(mgr); 508 509 idp = &mgr->ctx_handles; 510 511 idr_for_each_entry(idp, ctx, id) { 512 if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1) 513 DRM_ERROR("ctx %p is still alive\n", ctx); 514 } 515 516 idr_destroy(&mgr->ctx_handles); 517 mutex_destroy(&mgr->lock); 518 } 519