1 /* 2 * Copyright 2008 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 22 * DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: 25 * Jerome Glisse <glisse@freedesktop.org> 26 */ 27 28 #include <linux/file.h> 29 #include <linux/pagemap.h> 30 #include <linux/sync_file.h> 31 #include <linux/dma-buf.h> 32 33 #include <drm/amdgpu_drm.h> 34 #include <drm/drm_syncobj.h> 35 #include "amdgpu_cs.h" 36 #include "amdgpu.h" 37 #include "amdgpu_trace.h" 38 #include "amdgpu_gmc.h" 39 #include "amdgpu_gem.h" 40 #include "amdgpu_ras.h" 41 42 static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p, 43 struct drm_amdgpu_cs_chunk_fence *data, 44 uint32_t *offset) 45 { 46 struct drm_gem_object *gobj; 47 struct amdgpu_bo *bo; 48 unsigned long size; 49 int r; 50 51 gobj = drm_gem_object_lookup(p->filp, data->handle); 52 if (gobj == NULL) 53 return -EINVAL; 54 55 bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); 56 p->uf_entry.priority = 0; 57 p->uf_entry.tv.bo = &bo->tbo; 58 /* One for TTM and two for the CS job */ 59 p->uf_entry.tv.num_shared = 3; 60 61 drm_gem_object_put(gobj); 62 63 size = amdgpu_bo_size(bo); 64 if (size != PAGE_SIZE || (data->offset + 8) > size) { 65 r = -EINVAL; 66 goto error_unref; 67 } 68 69 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { 70 r = -EINVAL; 71 goto error_unref; 72 } 73 74 *offset = data->offset; 75 76 return 0; 77 78 error_unref: 79 amdgpu_bo_unref(&bo); 80 return r; 81 } 82 83 static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p, 84 struct drm_amdgpu_bo_list_in *data) 85 { 86 int r; 87 struct drm_amdgpu_bo_list_entry *info = NULL; 88 89 r = amdgpu_bo_create_list_entry_array(data, &info); 90 if (r) 91 return r; 92 93 r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number, 94 &p->bo_list); 95 if (r) 96 goto error_free; 97 98 kvfree(info); 99 return 0; 100 101 error_free: 102 kvfree(info); 103 104 return r; 105 } 106 107 static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs) 108 { 109 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 110 struct amdgpu_vm *vm = &fpriv->vm; 111 uint64_t *chunk_array_user; 112 uint64_t *chunk_array; 113 unsigned size, num_ibs = 0; 114 uint32_t uf_offset = 0; 115 int i; 116 int ret; 117 118 if (cs->in.num_chunks == 0) 119 return 0; 120 121 chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); 122 if (!chunk_array) 123 return -ENOMEM; 124 125 p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); 126 if (!p->ctx) { 127 ret = -EINVAL; 128 goto free_chunk; 129 } 130 131 mutex_lock(&p->ctx->lock); 132 133 /* skip guilty context job */ 134 if (atomic_read(&p->ctx->guilty) == 1) { 135 ret = -ECANCELED; 136 goto free_chunk; 137 } 138 139 /* get chunks */ 140 chunk_array_user = u64_to_user_ptr(cs->in.chunks); 141 if (copy_from_user(chunk_array, chunk_array_user, 142 sizeof(uint64_t)*cs->in.num_chunks)) { 143 ret = -EFAULT; 144 goto free_chunk; 145 } 146 147 p->nchunks = cs->in.num_chunks; 148 p->chunks = kvmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk), 149 GFP_KERNEL); 150 if (!p->chunks) { 151 ret = -ENOMEM; 152 goto free_chunk; 153 } 154 155 for (i = 0; i < p->nchunks; i++) { 156 struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL; 157 struct drm_amdgpu_cs_chunk user_chunk; 158 uint32_t __user *cdata; 159 160 chunk_ptr = u64_to_user_ptr(chunk_array[i]); 161 if (copy_from_user(&user_chunk, chunk_ptr, 162 sizeof(struct drm_amdgpu_cs_chunk))) { 163 ret = -EFAULT; 164 i--; 165 goto free_partial_kdata; 166 } 167 p->chunks[i].chunk_id = user_chunk.chunk_id; 168 p->chunks[i].length_dw = user_chunk.length_dw; 169 170 size = p->chunks[i].length_dw; 171 cdata = u64_to_user_ptr(user_chunk.chunk_data); 172 173 p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL); 174 if (p->chunks[i].kdata == NULL) { 175 ret = -ENOMEM; 176 i--; 177 goto free_partial_kdata; 178 } 179 size *= sizeof(uint32_t); 180 if (copy_from_user(p->chunks[i].kdata, cdata, size)) { 181 ret = -EFAULT; 182 goto free_partial_kdata; 183 } 184 185 switch (p->chunks[i].chunk_id) { 186 case AMDGPU_CHUNK_ID_IB: 187 ++num_ibs; 188 break; 189 190 case AMDGPU_CHUNK_ID_FENCE: 191 size = sizeof(struct drm_amdgpu_cs_chunk_fence); 192 if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { 193 ret = -EINVAL; 194 goto free_partial_kdata; 195 } 196 197 ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata, 198 &uf_offset); 199 if (ret) 200 goto free_partial_kdata; 201 202 break; 203 204 case AMDGPU_CHUNK_ID_BO_HANDLES: 205 size = sizeof(struct drm_amdgpu_bo_list_in); 206 if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { 207 ret = -EINVAL; 208 goto free_partial_kdata; 209 } 210 211 ret = amdgpu_cs_bo_handles_chunk(p, p->chunks[i].kdata); 212 if (ret) 213 goto free_partial_kdata; 214 215 break; 216 217 case AMDGPU_CHUNK_ID_DEPENDENCIES: 218 case AMDGPU_CHUNK_ID_SYNCOBJ_IN: 219 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: 220 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: 221 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: 222 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: 223 break; 224 225 default: 226 ret = -EINVAL; 227 goto free_partial_kdata; 228 } 229 } 230 231 ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm); 232 if (ret) 233 goto free_all_kdata; 234 235 if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) { 236 ret = -ECANCELED; 237 goto free_all_kdata; 238 } 239 240 if (p->uf_entry.tv.bo) 241 p->job->uf_addr = uf_offset; 242 kvfree(chunk_array); 243 244 /* Use this opportunity to fill in task info for the vm */ 245 amdgpu_vm_set_task_info(vm); 246 247 return 0; 248 249 free_all_kdata: 250 i = p->nchunks - 1; 251 free_partial_kdata: 252 for (; i >= 0; i--) 253 kvfree(p->chunks[i].kdata); 254 kvfree(p->chunks); 255 p->chunks = NULL; 256 p->nchunks = 0; 257 free_chunk: 258 kvfree(chunk_array); 259 260 return ret; 261 } 262 263 /* Convert microseconds to bytes. */ 264 static u64 us_to_bytes(struct amdgpu_device *adev, s64 us) 265 { 266 if (us <= 0 || !adev->mm_stats.log2_max_MBps) 267 return 0; 268 269 /* Since accum_us is incremented by a million per second, just 270 * multiply it by the number of MB/s to get the number of bytes. 271 */ 272 return us << adev->mm_stats.log2_max_MBps; 273 } 274 275 static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes) 276 { 277 if (!adev->mm_stats.log2_max_MBps) 278 return 0; 279 280 return bytes >> adev->mm_stats.log2_max_MBps; 281 } 282 283 /* Returns how many bytes TTM can move right now. If no bytes can be moved, 284 * it returns 0. If it returns non-zero, it's OK to move at least one buffer, 285 * which means it can go over the threshold once. If that happens, the driver 286 * will be in debt and no other buffer migrations can be done until that debt 287 * is repaid. 288 * 289 * This approach allows moving a buffer of any size (it's important to allow 290 * that). 291 * 292 * The currency is simply time in microseconds and it increases as the clock 293 * ticks. The accumulated microseconds (us) are converted to bytes and 294 * returned. 295 */ 296 static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev, 297 u64 *max_bytes, 298 u64 *max_vis_bytes) 299 { 300 s64 time_us, increment_us; 301 u64 free_vram, total_vram, used_vram; 302 /* Allow a maximum of 200 accumulated ms. This is basically per-IB 303 * throttling. 304 * 305 * It means that in order to get full max MBps, at least 5 IBs per 306 * second must be submitted and not more than 200ms apart from each 307 * other. 308 */ 309 const s64 us_upper_bound = 200000; 310 311 if (!adev->mm_stats.log2_max_MBps) { 312 *max_bytes = 0; 313 *max_vis_bytes = 0; 314 return; 315 } 316 317 total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size); 318 used_vram = ttm_resource_manager_usage(&adev->mman.vram_mgr.manager); 319 free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram; 320 321 spin_lock(&adev->mm_stats.lock); 322 323 /* Increase the amount of accumulated us. */ 324 time_us = ktime_to_us(ktime_get()); 325 increment_us = time_us - adev->mm_stats.last_update_us; 326 adev->mm_stats.last_update_us = time_us; 327 adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us, 328 us_upper_bound); 329 330 /* This prevents the short period of low performance when the VRAM 331 * usage is low and the driver is in debt or doesn't have enough 332 * accumulated us to fill VRAM quickly. 333 * 334 * The situation can occur in these cases: 335 * - a lot of VRAM is freed by userspace 336 * - the presence of a big buffer causes a lot of evictions 337 * (solution: split buffers into smaller ones) 338 * 339 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting 340 * accum_us to a positive number. 341 */ 342 if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) { 343 s64 min_us; 344 345 /* Be more aggressive on dGPUs. Try to fill a portion of free 346 * VRAM now. 347 */ 348 if (!(adev->flags & AMD_IS_APU)) 349 min_us = bytes_to_us(adev, free_vram / 4); 350 else 351 min_us = 0; /* Reset accum_us on APUs. */ 352 353 adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us); 354 } 355 356 /* This is set to 0 if the driver is in debt to disallow (optional) 357 * buffer moves. 358 */ 359 *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us); 360 361 /* Do the same for visible VRAM if half of it is free */ 362 if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) { 363 u64 total_vis_vram = adev->gmc.visible_vram_size; 364 u64 used_vis_vram = 365 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr); 366 367 if (used_vis_vram < total_vis_vram) { 368 u64 free_vis_vram = total_vis_vram - used_vis_vram; 369 adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis + 370 increment_us, us_upper_bound); 371 372 if (free_vis_vram >= total_vis_vram / 2) 373 adev->mm_stats.accum_us_vis = 374 max(bytes_to_us(adev, free_vis_vram / 2), 375 adev->mm_stats.accum_us_vis); 376 } 377 378 *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis); 379 } else { 380 *max_vis_bytes = 0; 381 } 382 383 spin_unlock(&adev->mm_stats.lock); 384 } 385 386 /* Report how many bytes have really been moved for the last command 387 * submission. This can result in a debt that can stop buffer migrations 388 * temporarily. 389 */ 390 void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, 391 u64 num_vis_bytes) 392 { 393 spin_lock(&adev->mm_stats.lock); 394 adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes); 395 adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes); 396 spin_unlock(&adev->mm_stats.lock); 397 } 398 399 static int amdgpu_cs_bo_validate(void *param, struct amdgpu_bo *bo) 400 { 401 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 402 struct amdgpu_cs_parser *p = param; 403 struct ttm_operation_ctx ctx = { 404 .interruptible = true, 405 .no_wait_gpu = false, 406 .resv = bo->tbo.base.resv 407 }; 408 uint32_t domain; 409 int r; 410 411 if (bo->tbo.pin_count) 412 return 0; 413 414 /* Don't move this buffer if we have depleted our allowance 415 * to move it. Don't move anything if the threshold is zero. 416 */ 417 if (p->bytes_moved < p->bytes_moved_threshold && 418 (!bo->tbo.base.dma_buf || 419 list_empty(&bo->tbo.base.dma_buf->attachments))) { 420 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 421 (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) { 422 /* And don't move a CPU_ACCESS_REQUIRED BO to limited 423 * visible VRAM if we've depleted our allowance to do 424 * that. 425 */ 426 if (p->bytes_moved_vis < p->bytes_moved_vis_threshold) 427 domain = bo->preferred_domains; 428 else 429 domain = bo->allowed_domains; 430 } else { 431 domain = bo->preferred_domains; 432 } 433 } else { 434 domain = bo->allowed_domains; 435 } 436 437 retry: 438 amdgpu_bo_placement_from_domain(bo, domain); 439 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 440 441 p->bytes_moved += ctx.bytes_moved; 442 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 443 amdgpu_bo_in_cpu_visible_vram(bo)) 444 p->bytes_moved_vis += ctx.bytes_moved; 445 446 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { 447 domain = bo->allowed_domains; 448 goto retry; 449 } 450 451 return r; 452 } 453 454 static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, 455 struct list_head *validated) 456 { 457 struct ttm_operation_ctx ctx = { true, false }; 458 struct amdgpu_bo_list_entry *lobj; 459 int r; 460 461 list_for_each_entry(lobj, validated, tv.head) { 462 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(lobj->tv.bo); 463 struct mm_struct *usermm; 464 465 usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); 466 if (usermm && usermm != current->mm) 467 return -EPERM; 468 469 if (amdgpu_ttm_tt_is_userptr(bo->tbo.ttm) && 470 lobj->user_invalidated && lobj->user_pages) { 471 amdgpu_bo_placement_from_domain(bo, 472 AMDGPU_GEM_DOMAIN_CPU); 473 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 474 if (r) 475 return r; 476 477 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, 478 lobj->user_pages); 479 } 480 481 r = amdgpu_cs_bo_validate(p, bo); 482 if (r) 483 return r; 484 485 kvfree(lobj->user_pages); 486 lobj->user_pages = NULL; 487 } 488 return 0; 489 } 490 491 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, 492 union drm_amdgpu_cs *cs) 493 { 494 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 495 struct amdgpu_vm *vm = &fpriv->vm; 496 struct amdgpu_bo_list_entry *e; 497 struct list_head duplicates; 498 struct amdgpu_bo *gds; 499 struct amdgpu_bo *gws; 500 struct amdgpu_bo *oa; 501 int r; 502 503 INIT_LIST_HEAD(&p->validated); 504 505 /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */ 506 if (cs->in.bo_list_handle) { 507 if (p->bo_list) 508 return -EINVAL; 509 510 r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle, 511 &p->bo_list); 512 if (r) 513 return r; 514 } else if (!p->bo_list) { 515 /* Create a empty bo_list when no handle is provided */ 516 r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0, 517 &p->bo_list); 518 if (r) 519 return r; 520 } 521 522 /* One for TTM and one for the CS job */ 523 amdgpu_bo_list_for_each_entry(e, p->bo_list) 524 e->tv.num_shared = 2; 525 526 amdgpu_bo_list_get_list(p->bo_list, &p->validated); 527 528 INIT_LIST_HEAD(&duplicates); 529 amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd); 530 531 if (p->uf_entry.tv.bo && !ttm_to_amdgpu_bo(p->uf_entry.tv.bo)->parent) 532 list_add(&p->uf_entry.tv.head, &p->validated); 533 534 /* Get userptr backing pages. If pages are updated after registered 535 * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do 536 * amdgpu_ttm_backend_bind() to flush and invalidate new pages 537 */ 538 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { 539 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); 540 bool userpage_invalidated = false; 541 int i; 542 543 e->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages, 544 sizeof(struct page *), 545 GFP_KERNEL | __GFP_ZERO); 546 if (!e->user_pages) { 547 DRM_ERROR("kvmalloc_array failure\n"); 548 return -ENOMEM; 549 } 550 551 r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages); 552 if (r) { 553 kvfree(e->user_pages); 554 e->user_pages = NULL; 555 return r; 556 } 557 558 for (i = 0; i < bo->tbo.ttm->num_pages; i++) { 559 if (bo->tbo.ttm->pages[i] != e->user_pages[i]) { 560 userpage_invalidated = true; 561 break; 562 } 563 } 564 e->user_invalidated = userpage_invalidated; 565 } 566 567 r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, 568 &duplicates); 569 if (unlikely(r != 0)) { 570 if (r != -ERESTARTSYS) 571 DRM_ERROR("ttm_eu_reserve_buffers failed.\n"); 572 goto out; 573 } 574 575 amdgpu_bo_list_for_each_entry(e, p->bo_list) { 576 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); 577 578 e->bo_va = amdgpu_vm_bo_find(vm, bo); 579 } 580 581 /* Move fence waiting after getting reservation lock of 582 * PD root. Then there is no need on a ctx mutex lock. 583 */ 584 r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entity); 585 if (unlikely(r != 0)) { 586 if (r != -ERESTARTSYS) 587 DRM_ERROR("amdgpu_ctx_wait_prev_fence failed.\n"); 588 goto error_validate; 589 } 590 591 amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold, 592 &p->bytes_moved_vis_threshold); 593 p->bytes_moved = 0; 594 p->bytes_moved_vis = 0; 595 596 r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm, 597 amdgpu_cs_bo_validate, p); 598 if (r) { 599 DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n"); 600 goto error_validate; 601 } 602 603 r = amdgpu_cs_list_validate(p, &duplicates); 604 if (r) 605 goto error_validate; 606 607 r = amdgpu_cs_list_validate(p, &p->validated); 608 if (r) 609 goto error_validate; 610 611 amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved, 612 p->bytes_moved_vis); 613 614 gds = p->bo_list->gds_obj; 615 gws = p->bo_list->gws_obj; 616 oa = p->bo_list->oa_obj; 617 618 if (gds) { 619 p->job->gds_base = amdgpu_bo_gpu_offset(gds) >> PAGE_SHIFT; 620 p->job->gds_size = amdgpu_bo_size(gds) >> PAGE_SHIFT; 621 } 622 if (gws) { 623 p->job->gws_base = amdgpu_bo_gpu_offset(gws) >> PAGE_SHIFT; 624 p->job->gws_size = amdgpu_bo_size(gws) >> PAGE_SHIFT; 625 } 626 if (oa) { 627 p->job->oa_base = amdgpu_bo_gpu_offset(oa) >> PAGE_SHIFT; 628 p->job->oa_size = amdgpu_bo_size(oa) >> PAGE_SHIFT; 629 } 630 631 if (!r && p->uf_entry.tv.bo) { 632 struct amdgpu_bo *uf = ttm_to_amdgpu_bo(p->uf_entry.tv.bo); 633 634 r = amdgpu_ttm_alloc_gart(&uf->tbo); 635 p->job->uf_addr += amdgpu_bo_gpu_offset(uf); 636 } 637 638 error_validate: 639 if (r) 640 ttm_eu_backoff_reservation(&p->ticket, &p->validated); 641 out: 642 return r; 643 } 644 645 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) 646 { 647 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 648 struct amdgpu_bo_list_entry *e; 649 int r; 650 651 list_for_each_entry(e, &p->validated, tv.head) { 652 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); 653 struct dma_resv *resv = bo->tbo.base.resv; 654 enum amdgpu_sync_mode sync_mode; 655 656 sync_mode = amdgpu_bo_explicit_sync(bo) ? 657 AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER; 658 r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, sync_mode, 659 &fpriv->vm); 660 if (r) 661 return r; 662 } 663 return 0; 664 } 665 666 /** 667 * amdgpu_cs_parser_fini() - clean parser states 668 * @parser: parser structure holding parsing context. 669 * @error: error number 670 * @backoff: indicator to backoff the reservation 671 * 672 * If error is set then unvalidate buffer, otherwise just free memory 673 * used by parsing context. 674 **/ 675 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, 676 bool backoff) 677 { 678 unsigned i; 679 680 if (error && backoff) 681 ttm_eu_backoff_reservation(&parser->ticket, 682 &parser->validated); 683 684 for (i = 0; i < parser->num_post_deps; i++) { 685 drm_syncobj_put(parser->post_deps[i].syncobj); 686 kfree(parser->post_deps[i].chain); 687 } 688 kfree(parser->post_deps); 689 690 dma_fence_put(parser->fence); 691 692 if (parser->ctx) { 693 mutex_unlock(&parser->ctx->lock); 694 amdgpu_ctx_put(parser->ctx); 695 } 696 if (parser->bo_list) 697 amdgpu_bo_list_put(parser->bo_list); 698 699 for (i = 0; i < parser->nchunks; i++) 700 kvfree(parser->chunks[i].kdata); 701 kvfree(parser->chunks); 702 if (parser->job) 703 amdgpu_job_free(parser->job); 704 if (parser->uf_entry.tv.bo) { 705 struct amdgpu_bo *uf = ttm_to_amdgpu_bo(parser->uf_entry.tv.bo); 706 707 amdgpu_bo_unref(&uf); 708 } 709 } 710 711 static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) 712 { 713 struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched); 714 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 715 struct amdgpu_device *adev = p->adev; 716 struct amdgpu_vm *vm = &fpriv->vm; 717 struct amdgpu_bo_list_entry *e; 718 struct amdgpu_bo_va *bo_va; 719 struct amdgpu_bo *bo; 720 int r; 721 722 /* Only for UVD/VCE VM emulation */ 723 if (ring->funcs->parse_cs || ring->funcs->patch_cs_in_place) { 724 unsigned i, j; 725 726 for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) { 727 struct drm_amdgpu_cs_chunk_ib *chunk_ib; 728 struct amdgpu_bo_va_mapping *m; 729 struct amdgpu_bo *aobj = NULL; 730 struct amdgpu_cs_chunk *chunk; 731 uint64_t offset, va_start; 732 struct amdgpu_ib *ib; 733 uint8_t *kptr; 734 735 chunk = &p->chunks[i]; 736 ib = &p->job->ibs[j]; 737 chunk_ib = chunk->kdata; 738 739 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) 740 continue; 741 742 va_start = chunk_ib->va_start & AMDGPU_GMC_HOLE_MASK; 743 r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m); 744 if (r) { 745 DRM_ERROR("IB va_start is invalid\n"); 746 return r; 747 } 748 749 if ((va_start + chunk_ib->ib_bytes) > 750 (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) { 751 DRM_ERROR("IB va_start+ib_bytes is invalid\n"); 752 return -EINVAL; 753 } 754 755 /* the IB should be reserved at this point */ 756 r = amdgpu_bo_kmap(aobj, (void **)&kptr); 757 if (r) { 758 return r; 759 } 760 761 offset = m->start * AMDGPU_GPU_PAGE_SIZE; 762 kptr += va_start - offset; 763 764 if (ring->funcs->parse_cs) { 765 memcpy(ib->ptr, kptr, chunk_ib->ib_bytes); 766 amdgpu_bo_kunmap(aobj); 767 768 r = amdgpu_ring_parse_cs(ring, p, p->job, ib); 769 if (r) 770 return r; 771 } else { 772 ib->ptr = (uint32_t *)kptr; 773 r = amdgpu_ring_patch_cs_in_place(ring, p, p->job, ib); 774 amdgpu_bo_kunmap(aobj); 775 if (r) 776 return r; 777 } 778 779 j++; 780 } 781 } 782 783 if (!p->job->vm) 784 return amdgpu_cs_sync_rings(p); 785 786 787 r = amdgpu_vm_clear_freed(adev, vm, NULL); 788 if (r) 789 return r; 790 791 r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); 792 if (r) 793 return r; 794 795 r = amdgpu_sync_fence(&p->job->sync, fpriv->prt_va->last_pt_update); 796 if (r) 797 return r; 798 799 if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { 800 bo_va = fpriv->csa_va; 801 BUG_ON(!bo_va); 802 r = amdgpu_vm_bo_update(adev, bo_va, false); 803 if (r) 804 return r; 805 806 r = amdgpu_sync_fence(&p->job->sync, bo_va->last_pt_update); 807 if (r) 808 return r; 809 } 810 811 amdgpu_bo_list_for_each_entry(e, p->bo_list) { 812 /* ignore duplicates */ 813 bo = ttm_to_amdgpu_bo(e->tv.bo); 814 if (!bo) 815 continue; 816 817 bo_va = e->bo_va; 818 if (bo_va == NULL) 819 continue; 820 821 r = amdgpu_vm_bo_update(adev, bo_va, false); 822 if (r) 823 return r; 824 825 r = amdgpu_sync_fence(&p->job->sync, bo_va->last_pt_update); 826 if (r) 827 return r; 828 } 829 830 r = amdgpu_vm_handle_moved(adev, vm); 831 if (r) 832 return r; 833 834 r = amdgpu_vm_update_pdes(adev, vm, false); 835 if (r) 836 return r; 837 838 r = amdgpu_sync_fence(&p->job->sync, vm->last_update); 839 if (r) 840 return r; 841 842 p->job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo); 843 844 if (amdgpu_vm_debug) { 845 /* Invalidate all BOs to test for userspace bugs */ 846 amdgpu_bo_list_for_each_entry(e, p->bo_list) { 847 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); 848 849 /* ignore duplicates */ 850 if (!bo) 851 continue; 852 853 amdgpu_vm_bo_invalidate(adev, bo, false); 854 } 855 } 856 857 return amdgpu_cs_sync_rings(p); 858 } 859 860 static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, 861 struct amdgpu_cs_parser *parser) 862 { 863 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 864 struct amdgpu_vm *vm = &fpriv->vm; 865 int r, ce_preempt = 0, de_preempt = 0; 866 struct amdgpu_ring *ring; 867 int i, j; 868 869 for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) { 870 struct amdgpu_cs_chunk *chunk; 871 struct amdgpu_ib *ib; 872 struct drm_amdgpu_cs_chunk_ib *chunk_ib; 873 struct drm_sched_entity *entity; 874 875 chunk = &parser->chunks[i]; 876 ib = &parser->job->ibs[j]; 877 chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata; 878 879 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) 880 continue; 881 882 if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && 883 (amdgpu_mcbp || amdgpu_sriov_vf(adev))) { 884 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) { 885 if (chunk_ib->flags & AMDGPU_IB_FLAG_CE) 886 ce_preempt++; 887 else 888 de_preempt++; 889 } 890 891 /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */ 892 if (ce_preempt > 1 || de_preempt > 1) 893 return -EINVAL; 894 } 895 896 r = amdgpu_ctx_get_entity(parser->ctx, chunk_ib->ip_type, 897 chunk_ib->ip_instance, chunk_ib->ring, 898 &entity); 899 if (r) 900 return r; 901 902 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE) 903 parser->job->preamble_status |= 904 AMDGPU_PREAMBLE_IB_PRESENT; 905 906 if (parser->entity && parser->entity != entity) 907 return -EINVAL; 908 909 /* Return if there is no run queue associated with this entity. 910 * Possibly because of disabled HW IP*/ 911 if (entity->rq == NULL) 912 return -EINVAL; 913 914 parser->entity = entity; 915 916 ring = to_amdgpu_ring(entity->rq->sched); 917 r = amdgpu_ib_get(adev, vm, ring->funcs->parse_cs ? 918 chunk_ib->ib_bytes : 0, 919 AMDGPU_IB_POOL_DELAYED, ib); 920 if (r) { 921 DRM_ERROR("Failed to get ib !\n"); 922 return r; 923 } 924 925 ib->gpu_addr = chunk_ib->va_start; 926 ib->length_dw = chunk_ib->ib_bytes / 4; 927 ib->flags = chunk_ib->flags; 928 929 j++; 930 } 931 932 /* MM engine doesn't support user fences */ 933 ring = to_amdgpu_ring(parser->entity->rq->sched); 934 if (parser->job->uf_addr && ring->funcs->no_user_fence) 935 return -EINVAL; 936 937 return 0; 938 } 939 940 static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p, 941 struct amdgpu_cs_chunk *chunk) 942 { 943 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 944 unsigned num_deps; 945 int i, r; 946 struct drm_amdgpu_cs_chunk_dep *deps; 947 948 deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata; 949 num_deps = chunk->length_dw * 4 / 950 sizeof(struct drm_amdgpu_cs_chunk_dep); 951 952 for (i = 0; i < num_deps; ++i) { 953 struct amdgpu_ctx *ctx; 954 struct drm_sched_entity *entity; 955 struct dma_fence *fence; 956 957 ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id); 958 if (ctx == NULL) 959 return -EINVAL; 960 961 r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type, 962 deps[i].ip_instance, 963 deps[i].ring, &entity); 964 if (r) { 965 amdgpu_ctx_put(ctx); 966 return r; 967 } 968 969 fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle); 970 amdgpu_ctx_put(ctx); 971 972 if (IS_ERR(fence)) 973 return PTR_ERR(fence); 974 else if (!fence) 975 continue; 976 977 if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) { 978 struct drm_sched_fence *s_fence; 979 struct dma_fence *old = fence; 980 981 s_fence = to_drm_sched_fence(fence); 982 fence = dma_fence_get(&s_fence->scheduled); 983 dma_fence_put(old); 984 } 985 986 r = amdgpu_sync_fence(&p->job->sync, fence); 987 dma_fence_put(fence); 988 if (r) 989 return r; 990 } 991 return 0; 992 } 993 994 static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p, 995 uint32_t handle, u64 point, 996 u64 flags) 997 { 998 struct dma_fence *fence; 999 int r; 1000 1001 r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence); 1002 if (r) { 1003 DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n", 1004 handle, point, r); 1005 return r; 1006 } 1007 1008 r = amdgpu_sync_fence(&p->job->sync, fence); 1009 dma_fence_put(fence); 1010 1011 return r; 1012 } 1013 1014 static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser *p, 1015 struct amdgpu_cs_chunk *chunk) 1016 { 1017 struct drm_amdgpu_cs_chunk_sem *deps; 1018 unsigned num_deps; 1019 int i, r; 1020 1021 deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; 1022 num_deps = chunk->length_dw * 4 / 1023 sizeof(struct drm_amdgpu_cs_chunk_sem); 1024 for (i = 0; i < num_deps; ++i) { 1025 r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle, 1026 0, 0); 1027 if (r) 1028 return r; 1029 } 1030 1031 return 0; 1032 } 1033 1034 1035 static int amdgpu_cs_process_syncobj_timeline_in_dep(struct amdgpu_cs_parser *p, 1036 struct amdgpu_cs_chunk *chunk) 1037 { 1038 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps; 1039 unsigned num_deps; 1040 int i, r; 1041 1042 syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata; 1043 num_deps = chunk->length_dw * 4 / 1044 sizeof(struct drm_amdgpu_cs_chunk_syncobj); 1045 for (i = 0; i < num_deps; ++i) { 1046 r = amdgpu_syncobj_lookup_and_add_to_sync(p, 1047 syncobj_deps[i].handle, 1048 syncobj_deps[i].point, 1049 syncobj_deps[i].flags); 1050 if (r) 1051 return r; 1052 } 1053 1054 return 0; 1055 } 1056 1057 static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p, 1058 struct amdgpu_cs_chunk *chunk) 1059 { 1060 struct drm_amdgpu_cs_chunk_sem *deps; 1061 unsigned num_deps; 1062 int i; 1063 1064 deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; 1065 num_deps = chunk->length_dw * 4 / 1066 sizeof(struct drm_amdgpu_cs_chunk_sem); 1067 1068 if (p->post_deps) 1069 return -EINVAL; 1070 1071 p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), 1072 GFP_KERNEL); 1073 p->num_post_deps = 0; 1074 1075 if (!p->post_deps) 1076 return -ENOMEM; 1077 1078 1079 for (i = 0; i < num_deps; ++i) { 1080 p->post_deps[i].syncobj = 1081 drm_syncobj_find(p->filp, deps[i].handle); 1082 if (!p->post_deps[i].syncobj) 1083 return -EINVAL; 1084 p->post_deps[i].chain = NULL; 1085 p->post_deps[i].point = 0; 1086 p->num_post_deps++; 1087 } 1088 1089 return 0; 1090 } 1091 1092 1093 static int amdgpu_cs_process_syncobj_timeline_out_dep(struct amdgpu_cs_parser *p, 1094 struct amdgpu_cs_chunk *chunk) 1095 { 1096 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps; 1097 unsigned num_deps; 1098 int i; 1099 1100 syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata; 1101 num_deps = chunk->length_dw * 4 / 1102 sizeof(struct drm_amdgpu_cs_chunk_syncobj); 1103 1104 if (p->post_deps) 1105 return -EINVAL; 1106 1107 p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), 1108 GFP_KERNEL); 1109 p->num_post_deps = 0; 1110 1111 if (!p->post_deps) 1112 return -ENOMEM; 1113 1114 for (i = 0; i < num_deps; ++i) { 1115 struct amdgpu_cs_post_dep *dep = &p->post_deps[i]; 1116 1117 dep->chain = NULL; 1118 if (syncobj_deps[i].point) { 1119 dep->chain = dma_fence_chain_alloc(); 1120 if (!dep->chain) 1121 return -ENOMEM; 1122 } 1123 1124 dep->syncobj = drm_syncobj_find(p->filp, 1125 syncobj_deps[i].handle); 1126 if (!dep->syncobj) { 1127 dma_fence_chain_free(dep->chain); 1128 return -EINVAL; 1129 } 1130 dep->point = syncobj_deps[i].point; 1131 p->num_post_deps++; 1132 } 1133 1134 return 0; 1135 } 1136 1137 static int amdgpu_cs_dependencies(struct amdgpu_device *adev, 1138 struct amdgpu_cs_parser *p) 1139 { 1140 int i, r; 1141 1142 /* TODO: Investigate why we still need the context lock */ 1143 mutex_unlock(&p->ctx->lock); 1144 1145 for (i = 0; i < p->nchunks; ++i) { 1146 struct amdgpu_cs_chunk *chunk; 1147 1148 chunk = &p->chunks[i]; 1149 1150 switch (chunk->chunk_id) { 1151 case AMDGPU_CHUNK_ID_DEPENDENCIES: 1152 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: 1153 r = amdgpu_cs_process_fence_dep(p, chunk); 1154 if (r) 1155 goto out; 1156 break; 1157 case AMDGPU_CHUNK_ID_SYNCOBJ_IN: 1158 r = amdgpu_cs_process_syncobj_in_dep(p, chunk); 1159 if (r) 1160 goto out; 1161 break; 1162 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: 1163 r = amdgpu_cs_process_syncobj_out_dep(p, chunk); 1164 if (r) 1165 goto out; 1166 break; 1167 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: 1168 r = amdgpu_cs_process_syncobj_timeline_in_dep(p, chunk); 1169 if (r) 1170 goto out; 1171 break; 1172 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: 1173 r = amdgpu_cs_process_syncobj_timeline_out_dep(p, chunk); 1174 if (r) 1175 goto out; 1176 break; 1177 } 1178 } 1179 1180 out: 1181 mutex_lock(&p->ctx->lock); 1182 return r; 1183 } 1184 1185 static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p) 1186 { 1187 int i; 1188 1189 for (i = 0; i < p->num_post_deps; ++i) { 1190 if (p->post_deps[i].chain && p->post_deps[i].point) { 1191 drm_syncobj_add_point(p->post_deps[i].syncobj, 1192 p->post_deps[i].chain, 1193 p->fence, p->post_deps[i].point); 1194 p->post_deps[i].chain = NULL; 1195 } else { 1196 drm_syncobj_replace_fence(p->post_deps[i].syncobj, 1197 p->fence); 1198 } 1199 } 1200 } 1201 1202 static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, 1203 union drm_amdgpu_cs *cs) 1204 { 1205 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 1206 struct drm_sched_entity *entity = p->entity; 1207 struct amdgpu_bo_list_entry *e; 1208 struct amdgpu_job *job; 1209 uint64_t seq; 1210 int r; 1211 1212 job = p->job; 1213 p->job = NULL; 1214 1215 r = drm_sched_job_init(&job->base, entity, &fpriv->vm); 1216 if (r) 1217 goto error_unlock; 1218 1219 drm_sched_job_arm(&job->base); 1220 1221 /* No memory allocation is allowed while holding the notifier lock. 1222 * The lock is held until amdgpu_cs_submit is finished and fence is 1223 * added to BOs. 1224 */ 1225 mutex_lock(&p->adev->notifier_lock); 1226 1227 /* If userptr are invalidated after amdgpu_cs_parser_bos(), return 1228 * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl. 1229 */ 1230 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { 1231 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); 1232 1233 r |= !amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); 1234 } 1235 if (r) { 1236 r = -EAGAIN; 1237 goto error_abort; 1238 } 1239 1240 p->fence = dma_fence_get(&job->base.s_fence->finished); 1241 1242 amdgpu_ctx_add_fence(p->ctx, entity, p->fence, &seq); 1243 amdgpu_cs_post_dependencies(p); 1244 1245 if ((job->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) && 1246 !p->ctx->preamble_presented) { 1247 job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST; 1248 p->ctx->preamble_presented = true; 1249 } 1250 1251 cs->out.handle = seq; 1252 job->uf_sequence = seq; 1253 1254 amdgpu_job_free_resources(job); 1255 1256 trace_amdgpu_cs_ioctl(job); 1257 amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->ticket); 1258 drm_sched_entity_push_job(&job->base); 1259 1260 amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm); 1261 1262 /* Make sure all BOs are remembered as writers */ 1263 amdgpu_bo_list_for_each_entry(e, p->bo_list) 1264 e->tv.num_shared = 0; 1265 1266 ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence); 1267 mutex_unlock(&p->adev->notifier_lock); 1268 1269 return 0; 1270 1271 error_abort: 1272 drm_sched_job_cleanup(&job->base); 1273 mutex_unlock(&p->adev->notifier_lock); 1274 1275 error_unlock: 1276 amdgpu_job_free(job); 1277 return r; 1278 } 1279 1280 static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *parser) 1281 { 1282 int i; 1283 1284 if (!trace_amdgpu_cs_enabled()) 1285 return; 1286 1287 for (i = 0; i < parser->job->num_ibs; i++) 1288 trace_amdgpu_cs(parser, i); 1289 } 1290 1291 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 1292 { 1293 struct amdgpu_device *adev = drm_to_adev(dev); 1294 union drm_amdgpu_cs *cs = data; 1295 struct amdgpu_cs_parser parser = {}; 1296 bool reserved_buffers = false; 1297 int r; 1298 1299 if (amdgpu_ras_intr_triggered()) 1300 return -EHWPOISON; 1301 1302 if (!adev->accel_working) 1303 return -EBUSY; 1304 1305 parser.adev = adev; 1306 parser.filp = filp; 1307 1308 r = amdgpu_cs_parser_init(&parser, data); 1309 if (r) { 1310 if (printk_ratelimit()) 1311 DRM_ERROR("Failed to initialize parser %d!\n", r); 1312 goto out; 1313 } 1314 1315 r = amdgpu_cs_ib_fill(adev, &parser); 1316 if (r) 1317 goto out; 1318 1319 r = amdgpu_cs_dependencies(adev, &parser); 1320 if (r) { 1321 DRM_ERROR("Failed in the dependencies handling %d!\n", r); 1322 goto out; 1323 } 1324 1325 r = amdgpu_cs_parser_bos(&parser, data); 1326 if (r) { 1327 if (r == -ENOMEM) 1328 DRM_ERROR("Not enough memory for command submission!\n"); 1329 else if (r != -ERESTARTSYS && r != -EAGAIN) 1330 DRM_ERROR("Failed to process the buffer list %d!\n", r); 1331 goto out; 1332 } 1333 1334 reserved_buffers = true; 1335 1336 trace_amdgpu_cs_ibs(&parser); 1337 1338 r = amdgpu_cs_vm_handling(&parser); 1339 if (r) 1340 goto out; 1341 1342 r = amdgpu_cs_submit(&parser, cs); 1343 1344 out: 1345 amdgpu_cs_parser_fini(&parser, r, reserved_buffers); 1346 1347 return r; 1348 } 1349 1350 /** 1351 * amdgpu_cs_wait_ioctl - wait for a command submission to finish 1352 * 1353 * @dev: drm device 1354 * @data: data from userspace 1355 * @filp: file private 1356 * 1357 * Wait for the command submission identified by handle to finish. 1358 */ 1359 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, 1360 struct drm_file *filp) 1361 { 1362 union drm_amdgpu_wait_cs *wait = data; 1363 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout); 1364 struct drm_sched_entity *entity; 1365 struct amdgpu_ctx *ctx; 1366 struct dma_fence *fence; 1367 long r; 1368 1369 ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id); 1370 if (ctx == NULL) 1371 return -EINVAL; 1372 1373 r = amdgpu_ctx_get_entity(ctx, wait->in.ip_type, wait->in.ip_instance, 1374 wait->in.ring, &entity); 1375 if (r) { 1376 amdgpu_ctx_put(ctx); 1377 return r; 1378 } 1379 1380 fence = amdgpu_ctx_get_fence(ctx, entity, wait->in.handle); 1381 if (IS_ERR(fence)) 1382 r = PTR_ERR(fence); 1383 else if (fence) { 1384 r = dma_fence_wait_timeout(fence, true, timeout); 1385 if (r > 0 && fence->error) 1386 r = fence->error; 1387 dma_fence_put(fence); 1388 } else 1389 r = 1; 1390 1391 amdgpu_ctx_put(ctx); 1392 if (r < 0) 1393 return r; 1394 1395 memset(wait, 0, sizeof(*wait)); 1396 wait->out.status = (r == 0); 1397 1398 return 0; 1399 } 1400 1401 /** 1402 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence 1403 * 1404 * @adev: amdgpu device 1405 * @filp: file private 1406 * @user: drm_amdgpu_fence copied from user space 1407 */ 1408 static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev, 1409 struct drm_file *filp, 1410 struct drm_amdgpu_fence *user) 1411 { 1412 struct drm_sched_entity *entity; 1413 struct amdgpu_ctx *ctx; 1414 struct dma_fence *fence; 1415 int r; 1416 1417 ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id); 1418 if (ctx == NULL) 1419 return ERR_PTR(-EINVAL); 1420 1421 r = amdgpu_ctx_get_entity(ctx, user->ip_type, user->ip_instance, 1422 user->ring, &entity); 1423 if (r) { 1424 amdgpu_ctx_put(ctx); 1425 return ERR_PTR(r); 1426 } 1427 1428 fence = amdgpu_ctx_get_fence(ctx, entity, user->seq_no); 1429 amdgpu_ctx_put(ctx); 1430 1431 return fence; 1432 } 1433 1434 int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data, 1435 struct drm_file *filp) 1436 { 1437 struct amdgpu_device *adev = drm_to_adev(dev); 1438 union drm_amdgpu_fence_to_handle *info = data; 1439 struct dma_fence *fence; 1440 struct drm_syncobj *syncobj; 1441 struct sync_file *sync_file; 1442 int fd, r; 1443 1444 fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence); 1445 if (IS_ERR(fence)) 1446 return PTR_ERR(fence); 1447 1448 if (!fence) 1449 fence = dma_fence_get_stub(); 1450 1451 switch (info->in.what) { 1452 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ: 1453 r = drm_syncobj_create(&syncobj, 0, fence); 1454 dma_fence_put(fence); 1455 if (r) 1456 return r; 1457 r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle); 1458 drm_syncobj_put(syncobj); 1459 return r; 1460 1461 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD: 1462 r = drm_syncobj_create(&syncobj, 0, fence); 1463 dma_fence_put(fence); 1464 if (r) 1465 return r; 1466 r = drm_syncobj_get_fd(syncobj, (int *)&info->out.handle); 1467 drm_syncobj_put(syncobj); 1468 return r; 1469 1470 case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD: 1471 fd = get_unused_fd_flags(O_CLOEXEC); 1472 if (fd < 0) { 1473 dma_fence_put(fence); 1474 return fd; 1475 } 1476 1477 sync_file = sync_file_create(fence); 1478 dma_fence_put(fence); 1479 if (!sync_file) { 1480 put_unused_fd(fd); 1481 return -ENOMEM; 1482 } 1483 1484 fd_install(fd, sync_file->file); 1485 info->out.handle = fd; 1486 return 0; 1487 1488 default: 1489 dma_fence_put(fence); 1490 return -EINVAL; 1491 } 1492 } 1493 1494 /** 1495 * amdgpu_cs_wait_all_fences - wait on all fences to signal 1496 * 1497 * @adev: amdgpu device 1498 * @filp: file private 1499 * @wait: wait parameters 1500 * @fences: array of drm_amdgpu_fence 1501 */ 1502 static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev, 1503 struct drm_file *filp, 1504 union drm_amdgpu_wait_fences *wait, 1505 struct drm_amdgpu_fence *fences) 1506 { 1507 uint32_t fence_count = wait->in.fence_count; 1508 unsigned int i; 1509 long r = 1; 1510 1511 for (i = 0; i < fence_count; i++) { 1512 struct dma_fence *fence; 1513 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); 1514 1515 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); 1516 if (IS_ERR(fence)) 1517 return PTR_ERR(fence); 1518 else if (!fence) 1519 continue; 1520 1521 r = dma_fence_wait_timeout(fence, true, timeout); 1522 dma_fence_put(fence); 1523 if (r < 0) 1524 return r; 1525 1526 if (r == 0) 1527 break; 1528 1529 if (fence->error) 1530 return fence->error; 1531 } 1532 1533 memset(wait, 0, sizeof(*wait)); 1534 wait->out.status = (r > 0); 1535 1536 return 0; 1537 } 1538 1539 /** 1540 * amdgpu_cs_wait_any_fence - wait on any fence to signal 1541 * 1542 * @adev: amdgpu device 1543 * @filp: file private 1544 * @wait: wait parameters 1545 * @fences: array of drm_amdgpu_fence 1546 */ 1547 static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev, 1548 struct drm_file *filp, 1549 union drm_amdgpu_wait_fences *wait, 1550 struct drm_amdgpu_fence *fences) 1551 { 1552 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); 1553 uint32_t fence_count = wait->in.fence_count; 1554 uint32_t first = ~0; 1555 struct dma_fence **array; 1556 unsigned int i; 1557 long r; 1558 1559 /* Prepare the fence array */ 1560 array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL); 1561 1562 if (array == NULL) 1563 return -ENOMEM; 1564 1565 for (i = 0; i < fence_count; i++) { 1566 struct dma_fence *fence; 1567 1568 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); 1569 if (IS_ERR(fence)) { 1570 r = PTR_ERR(fence); 1571 goto err_free_fence_array; 1572 } else if (fence) { 1573 array[i] = fence; 1574 } else { /* NULL, the fence has been already signaled */ 1575 r = 1; 1576 first = i; 1577 goto out; 1578 } 1579 } 1580 1581 r = dma_fence_wait_any_timeout(array, fence_count, true, timeout, 1582 &first); 1583 if (r < 0) 1584 goto err_free_fence_array; 1585 1586 out: 1587 memset(wait, 0, sizeof(*wait)); 1588 wait->out.status = (r > 0); 1589 wait->out.first_signaled = first; 1590 1591 if (first < fence_count && array[first]) 1592 r = array[first]->error; 1593 else 1594 r = 0; 1595 1596 err_free_fence_array: 1597 for (i = 0; i < fence_count; i++) 1598 dma_fence_put(array[i]); 1599 kfree(array); 1600 1601 return r; 1602 } 1603 1604 /** 1605 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish 1606 * 1607 * @dev: drm device 1608 * @data: data from userspace 1609 * @filp: file private 1610 */ 1611 int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data, 1612 struct drm_file *filp) 1613 { 1614 struct amdgpu_device *adev = drm_to_adev(dev); 1615 union drm_amdgpu_wait_fences *wait = data; 1616 uint32_t fence_count = wait->in.fence_count; 1617 struct drm_amdgpu_fence *fences_user; 1618 struct drm_amdgpu_fence *fences; 1619 int r; 1620 1621 /* Get the fences from userspace */ 1622 fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence), 1623 GFP_KERNEL); 1624 if (fences == NULL) 1625 return -ENOMEM; 1626 1627 fences_user = u64_to_user_ptr(wait->in.fences); 1628 if (copy_from_user(fences, fences_user, 1629 sizeof(struct drm_amdgpu_fence) * fence_count)) { 1630 r = -EFAULT; 1631 goto err_free_fences; 1632 } 1633 1634 if (wait->in.wait_all) 1635 r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences); 1636 else 1637 r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences); 1638 1639 err_free_fences: 1640 kfree(fences); 1641 1642 return r; 1643 } 1644 1645 /** 1646 * amdgpu_cs_find_mapping - find bo_va for VM address 1647 * 1648 * @parser: command submission parser context 1649 * @addr: VM address 1650 * @bo: resulting BO of the mapping found 1651 * @map: Placeholder to return found BO mapping 1652 * 1653 * Search the buffer objects in the command submission context for a certain 1654 * virtual memory address. Returns allocation structure when found, NULL 1655 * otherwise. 1656 */ 1657 int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser, 1658 uint64_t addr, struct amdgpu_bo **bo, 1659 struct amdgpu_bo_va_mapping **map) 1660 { 1661 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 1662 struct ttm_operation_ctx ctx = { false, false }; 1663 struct amdgpu_vm *vm = &fpriv->vm; 1664 struct amdgpu_bo_va_mapping *mapping; 1665 int r; 1666 1667 addr /= AMDGPU_GPU_PAGE_SIZE; 1668 1669 mapping = amdgpu_vm_bo_lookup_mapping(vm, addr); 1670 if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo) 1671 return -EINVAL; 1672 1673 *bo = mapping->bo_va->base.bo; 1674 *map = mapping; 1675 1676 /* Double check that the BO is reserved by this CS */ 1677 if (dma_resv_locking_ctx((*bo)->tbo.base.resv) != &parser->ticket) 1678 return -EINVAL; 1679 1680 if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) { 1681 (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 1682 amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains); 1683 r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx); 1684 if (r) 1685 return r; 1686 } 1687 1688 return amdgpu_ttm_alloc_gart(&(*bo)->tbo); 1689 } 1690