1 /* 2 * Copyright 2015 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: monk liu <monk.liu@amd.com> 23 */ 24 25 #include <drm/drmP.h> 26 #include <drm/drm_auth.h> 27 #include "amdgpu.h" 28 #include "amdgpu_sched.h" 29 30 static int amdgpu_ctx_priority_permit(struct drm_file *filp, 31 enum drm_sched_priority priority) 32 { 33 /* NORMAL and below are accessible by everyone */ 34 if (priority <= DRM_SCHED_PRIORITY_NORMAL) 35 return 0; 36 37 if (capable(CAP_SYS_NICE)) 38 return 0; 39 40 if (drm_is_current_master(filp)) 41 return 0; 42 43 return -EACCES; 44 } 45 46 static int amdgpu_ctx_init(struct amdgpu_device *adev, 47 enum drm_sched_priority priority, 48 struct drm_file *filp, 49 struct amdgpu_ctx *ctx) 50 { 51 unsigned i, j; 52 int r; 53 54 if (priority < 0 || priority >= DRM_SCHED_PRIORITY_MAX) 55 return -EINVAL; 56 57 r = amdgpu_ctx_priority_permit(filp, priority); 58 if (r) 59 return r; 60 61 memset(ctx, 0, sizeof(*ctx)); 62 ctx->adev = adev; 63 kref_init(&ctx->refcount); 64 spin_lock_init(&ctx->ring_lock); 65 ctx->fences = kcalloc(amdgpu_sched_jobs * AMDGPU_MAX_RINGS, 66 sizeof(struct dma_fence*), GFP_KERNEL); 67 if (!ctx->fences) 68 return -ENOMEM; 69 70 mutex_init(&ctx->lock); 71 72 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 73 ctx->rings[i].sequence = 1; 74 ctx->rings[i].fences = &ctx->fences[amdgpu_sched_jobs * i]; 75 } 76 77 ctx->reset_counter = atomic_read(&adev->gpu_reset_counter); 78 ctx->reset_counter_query = ctx->reset_counter; 79 ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter); 80 ctx->init_priority = priority; 81 ctx->override_priority = DRM_SCHED_PRIORITY_UNSET; 82 83 /* create context entity for each ring */ 84 for (i = 0; i < adev->num_rings; i++) { 85 struct amdgpu_ring *ring = adev->rings[i]; 86 struct drm_sched_rq *rq; 87 88 rq = &ring->sched.sched_rq[priority]; 89 90 if (ring == &adev->gfx.kiq.ring) 91 continue; 92 93 r = drm_sched_entity_init(&ring->sched, &ctx->rings[i].entity, 94 rq, amdgpu_sched_jobs, &ctx->guilty); 95 if (r) 96 goto failed; 97 } 98 99 r = amdgpu_queue_mgr_init(adev, &ctx->queue_mgr); 100 if (r) 101 goto failed; 102 103 return 0; 104 105 failed: 106 for (j = 0; j < i; j++) 107 drm_sched_entity_fini(&adev->rings[j]->sched, 108 &ctx->rings[j].entity); 109 kfree(ctx->fences); 110 ctx->fences = NULL; 111 return r; 112 } 113 114 static void amdgpu_ctx_fini(struct amdgpu_ctx *ctx) 115 { 116 struct amdgpu_device *adev = ctx->adev; 117 unsigned i, j; 118 119 if (!adev) 120 return; 121 122 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) 123 for (j = 0; j < amdgpu_sched_jobs; ++j) 124 dma_fence_put(ctx->rings[i].fences[j]); 125 kfree(ctx->fences); 126 ctx->fences = NULL; 127 128 for (i = 0; i < adev->num_rings; i++) 129 drm_sched_entity_fini(&adev->rings[i]->sched, 130 &ctx->rings[i].entity); 131 132 amdgpu_queue_mgr_fini(adev, &ctx->queue_mgr); 133 134 mutex_destroy(&ctx->lock); 135 } 136 137 static int amdgpu_ctx_alloc(struct amdgpu_device *adev, 138 struct amdgpu_fpriv *fpriv, 139 struct drm_file *filp, 140 enum drm_sched_priority priority, 141 uint32_t *id) 142 { 143 struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; 144 struct amdgpu_ctx *ctx; 145 int r; 146 147 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); 148 if (!ctx) 149 return -ENOMEM; 150 151 mutex_lock(&mgr->lock); 152 r = idr_alloc(&mgr->ctx_handles, ctx, 1, 0, GFP_KERNEL); 153 if (r < 0) { 154 mutex_unlock(&mgr->lock); 155 kfree(ctx); 156 return r; 157 } 158 159 *id = (uint32_t)r; 160 r = amdgpu_ctx_init(adev, priority, filp, ctx); 161 if (r) { 162 idr_remove(&mgr->ctx_handles, *id); 163 *id = 0; 164 kfree(ctx); 165 } 166 mutex_unlock(&mgr->lock); 167 return r; 168 } 169 170 static void amdgpu_ctx_do_release(struct kref *ref) 171 { 172 struct amdgpu_ctx *ctx; 173 174 ctx = container_of(ref, struct amdgpu_ctx, refcount); 175 176 amdgpu_ctx_fini(ctx); 177 178 kfree(ctx); 179 } 180 181 static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id) 182 { 183 struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; 184 struct amdgpu_ctx *ctx; 185 186 mutex_lock(&mgr->lock); 187 ctx = idr_remove(&mgr->ctx_handles, id); 188 if (ctx) 189 kref_put(&ctx->refcount, amdgpu_ctx_do_release); 190 mutex_unlock(&mgr->lock); 191 return ctx ? 0 : -EINVAL; 192 } 193 194 static int amdgpu_ctx_query(struct amdgpu_device *adev, 195 struct amdgpu_fpriv *fpriv, uint32_t id, 196 union drm_amdgpu_ctx_out *out) 197 { 198 struct amdgpu_ctx *ctx; 199 struct amdgpu_ctx_mgr *mgr; 200 unsigned reset_counter; 201 202 if (!fpriv) 203 return -EINVAL; 204 205 mgr = &fpriv->ctx_mgr; 206 mutex_lock(&mgr->lock); 207 ctx = idr_find(&mgr->ctx_handles, id); 208 if (!ctx) { 209 mutex_unlock(&mgr->lock); 210 return -EINVAL; 211 } 212 213 /* TODO: these two are always zero */ 214 out->state.flags = 0x0; 215 out->state.hangs = 0x0; 216 217 /* determine if a GPU reset has occured since the last call */ 218 reset_counter = atomic_read(&adev->gpu_reset_counter); 219 /* TODO: this should ideally return NO, GUILTY, or INNOCENT. */ 220 if (ctx->reset_counter_query == reset_counter) 221 out->state.reset_status = AMDGPU_CTX_NO_RESET; 222 else 223 out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET; 224 ctx->reset_counter_query = reset_counter; 225 226 mutex_unlock(&mgr->lock); 227 return 0; 228 } 229 230 static int amdgpu_ctx_query2(struct amdgpu_device *adev, 231 struct amdgpu_fpriv *fpriv, uint32_t id, 232 union drm_amdgpu_ctx_out *out) 233 { 234 struct amdgpu_ctx *ctx; 235 struct amdgpu_ctx_mgr *mgr; 236 237 if (!fpriv) 238 return -EINVAL; 239 240 mgr = &fpriv->ctx_mgr; 241 mutex_lock(&mgr->lock); 242 ctx = idr_find(&mgr->ctx_handles, id); 243 if (!ctx) { 244 mutex_unlock(&mgr->lock); 245 return -EINVAL; 246 } 247 248 out->state.flags = 0x0; 249 out->state.hangs = 0x0; 250 251 if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter)) 252 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET; 253 254 if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) 255 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST; 256 257 if (atomic_read(&ctx->guilty)) 258 out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY; 259 260 mutex_unlock(&mgr->lock); 261 return 0; 262 } 263 264 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, 265 struct drm_file *filp) 266 { 267 int r; 268 uint32_t id; 269 enum drm_sched_priority priority; 270 271 union drm_amdgpu_ctx *args = data; 272 struct amdgpu_device *adev = dev->dev_private; 273 struct amdgpu_fpriv *fpriv = filp->driver_priv; 274 275 r = 0; 276 id = args->in.ctx_id; 277 priority = amdgpu_to_sched_priority(args->in.priority); 278 279 /* For backwards compatibility reasons, we need to accept 280 * ioctls with garbage in the priority field */ 281 if (priority == DRM_SCHED_PRIORITY_INVALID) 282 priority = DRM_SCHED_PRIORITY_NORMAL; 283 284 switch (args->in.op) { 285 case AMDGPU_CTX_OP_ALLOC_CTX: 286 r = amdgpu_ctx_alloc(adev, fpriv, filp, priority, &id); 287 args->out.alloc.ctx_id = id; 288 break; 289 case AMDGPU_CTX_OP_FREE_CTX: 290 r = amdgpu_ctx_free(fpriv, id); 291 break; 292 case AMDGPU_CTX_OP_QUERY_STATE: 293 r = amdgpu_ctx_query(adev, fpriv, id, &args->out); 294 break; 295 case AMDGPU_CTX_OP_QUERY_STATE2: 296 r = amdgpu_ctx_query2(adev, fpriv, id, &args->out); 297 break; 298 default: 299 return -EINVAL; 300 } 301 302 return r; 303 } 304 305 struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id) 306 { 307 struct amdgpu_ctx *ctx; 308 struct amdgpu_ctx_mgr *mgr; 309 310 if (!fpriv) 311 return NULL; 312 313 mgr = &fpriv->ctx_mgr; 314 315 mutex_lock(&mgr->lock); 316 ctx = idr_find(&mgr->ctx_handles, id); 317 if (ctx) 318 kref_get(&ctx->refcount); 319 mutex_unlock(&mgr->lock); 320 return ctx; 321 } 322 323 int amdgpu_ctx_put(struct amdgpu_ctx *ctx) 324 { 325 if (ctx == NULL) 326 return -EINVAL; 327 328 kref_put(&ctx->refcount, amdgpu_ctx_do_release); 329 return 0; 330 } 331 332 int amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, 333 struct dma_fence *fence, uint64_t* handler) 334 { 335 struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx]; 336 uint64_t seq = cring->sequence; 337 unsigned idx = 0; 338 struct dma_fence *other = NULL; 339 340 idx = seq & (amdgpu_sched_jobs - 1); 341 other = cring->fences[idx]; 342 if (other) 343 BUG_ON(!dma_fence_is_signaled(other)); 344 345 dma_fence_get(fence); 346 347 spin_lock(&ctx->ring_lock); 348 cring->fences[idx] = fence; 349 cring->sequence++; 350 spin_unlock(&ctx->ring_lock); 351 352 dma_fence_put(other); 353 if (handler) 354 *handler = seq; 355 356 return 0; 357 } 358 359 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, 360 struct amdgpu_ring *ring, uint64_t seq) 361 { 362 struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx]; 363 struct dma_fence *fence; 364 365 spin_lock(&ctx->ring_lock); 366 367 if (seq == ~0ull) 368 seq = ctx->rings[ring->idx].sequence - 1; 369 370 if (seq >= cring->sequence) { 371 spin_unlock(&ctx->ring_lock); 372 return ERR_PTR(-EINVAL); 373 } 374 375 376 if (seq + amdgpu_sched_jobs < cring->sequence) { 377 spin_unlock(&ctx->ring_lock); 378 return NULL; 379 } 380 381 fence = dma_fence_get(cring->fences[seq & (amdgpu_sched_jobs - 1)]); 382 spin_unlock(&ctx->ring_lock); 383 384 return fence; 385 } 386 387 void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx, 388 enum drm_sched_priority priority) 389 { 390 int i; 391 struct amdgpu_device *adev = ctx->adev; 392 struct drm_sched_rq *rq; 393 struct drm_sched_entity *entity; 394 struct amdgpu_ring *ring; 395 enum drm_sched_priority ctx_prio; 396 397 ctx->override_priority = priority; 398 399 ctx_prio = (ctx->override_priority == DRM_SCHED_PRIORITY_UNSET) ? 400 ctx->init_priority : ctx->override_priority; 401 402 for (i = 0; i < adev->num_rings; i++) { 403 ring = adev->rings[i]; 404 entity = &ctx->rings[i].entity; 405 rq = &ring->sched.sched_rq[ctx_prio]; 406 407 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 408 continue; 409 410 drm_sched_entity_set_rq(entity, rq); 411 } 412 } 413 414 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx, unsigned ring_id) 415 { 416 struct amdgpu_ctx_ring *cring = &ctx->rings[ring_id]; 417 unsigned idx = cring->sequence & (amdgpu_sched_jobs - 1); 418 struct dma_fence *other = cring->fences[idx]; 419 420 if (other) { 421 signed long r; 422 r = dma_fence_wait(other, true); 423 if (r < 0) { 424 if (r != -ERESTARTSYS) 425 DRM_ERROR("Error (%ld) waiting for fence!\n", r); 426 427 return r; 428 } 429 } 430 431 return 0; 432 } 433 434 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr) 435 { 436 mutex_init(&mgr->lock); 437 idr_init(&mgr->ctx_handles); 438 } 439 440 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr) 441 { 442 struct amdgpu_ctx *ctx; 443 struct idr *idp; 444 uint32_t id; 445 446 idp = &mgr->ctx_handles; 447 448 idr_for_each_entry(idp, ctx, id) { 449 if (kref_put(&ctx->refcount, amdgpu_ctx_do_release) != 1) 450 DRM_ERROR("ctx %p is still alive\n", ctx); 451 } 452 453 idr_destroy(&mgr->ctx_handles); 454 mutex_destroy(&mgr->lock); 455 } 456