1 /* 2 * Copyright 2008 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 22 * DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: 25 * Jerome Glisse <glisse@freedesktop.org> 26 */ 27 28 #include <linux/file.h> 29 #include <linux/pagemap.h> 30 #include <linux/sync_file.h> 31 #include <linux/dma-buf.h> 32 33 #include <drm/amdgpu_drm.h> 34 #include <drm/drm_syncobj.h> 35 #include "amdgpu.h" 36 #include "amdgpu_trace.h" 37 #include "amdgpu_gmc.h" 38 #include "amdgpu_gem.h" 39 #include "amdgpu_ras.h" 40 41 static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p, 42 struct drm_amdgpu_cs_chunk_fence *data, 43 uint32_t *offset) 44 { 45 struct drm_gem_object *gobj; 46 struct amdgpu_bo *bo; 47 unsigned long size; 48 int r; 49 50 gobj = drm_gem_object_lookup(p->filp, data->handle); 51 if (gobj == NULL) 52 return -EINVAL; 53 54 bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); 55 p->uf_entry.priority = 0; 56 p->uf_entry.tv.bo = &bo->tbo; 57 /* One for TTM and one for the CS job */ 58 p->uf_entry.tv.num_shared = 2; 59 60 drm_gem_object_put(gobj); 61 62 size = amdgpu_bo_size(bo); 63 if (size != PAGE_SIZE || (data->offset + 8) > size) { 64 r = -EINVAL; 65 goto error_unref; 66 } 67 68 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { 69 r = -EINVAL; 70 goto error_unref; 71 } 72 73 *offset = data->offset; 74 75 return 0; 76 77 error_unref: 78 amdgpu_bo_unref(&bo); 79 return r; 80 } 81 82 static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p, 83 struct drm_amdgpu_bo_list_in *data) 84 { 85 int r; 86 struct drm_amdgpu_bo_list_entry *info = NULL; 87 88 r = amdgpu_bo_create_list_entry_array(data, &info); 89 if (r) 90 return r; 91 92 r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number, 93 &p->bo_list); 94 if (r) 95 goto error_free; 96 97 kvfree(info); 98 return 0; 99 100 error_free: 101 kvfree(info); 102 103 return r; 104 } 105 106 static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs) 107 { 108 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 109 struct amdgpu_vm *vm = &fpriv->vm; 110 uint64_t *chunk_array_user; 111 uint64_t *chunk_array; 112 unsigned size, num_ibs = 0; 113 uint32_t uf_offset = 0; 114 int i; 115 int ret; 116 117 if (cs->in.num_chunks == 0) 118 return 0; 119 120 chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); 121 if (!chunk_array) 122 return -ENOMEM; 123 124 p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); 125 if (!p->ctx) { 126 ret = -EINVAL; 127 goto free_chunk; 128 } 129 130 mutex_lock(&p->ctx->lock); 131 132 /* skip guilty context job */ 133 if (atomic_read(&p->ctx->guilty) == 1) { 134 ret = -ECANCELED; 135 goto free_chunk; 136 } 137 138 /* get chunks */ 139 chunk_array_user = u64_to_user_ptr(cs->in.chunks); 140 if (copy_from_user(chunk_array, chunk_array_user, 141 sizeof(uint64_t)*cs->in.num_chunks)) { 142 ret = -EFAULT; 143 goto free_chunk; 144 } 145 146 p->nchunks = cs->in.num_chunks; 147 p->chunks = kvmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk), 148 GFP_KERNEL); 149 if (!p->chunks) { 150 ret = -ENOMEM; 151 goto free_chunk; 152 } 153 154 for (i = 0; i < p->nchunks; i++) { 155 struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL; 156 struct drm_amdgpu_cs_chunk user_chunk; 157 uint32_t __user *cdata; 158 159 chunk_ptr = u64_to_user_ptr(chunk_array[i]); 160 if (copy_from_user(&user_chunk, chunk_ptr, 161 sizeof(struct drm_amdgpu_cs_chunk))) { 162 ret = -EFAULT; 163 i--; 164 goto free_partial_kdata; 165 } 166 p->chunks[i].chunk_id = user_chunk.chunk_id; 167 p->chunks[i].length_dw = user_chunk.length_dw; 168 169 size = p->chunks[i].length_dw; 170 cdata = u64_to_user_ptr(user_chunk.chunk_data); 171 172 p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL); 173 if (p->chunks[i].kdata == NULL) { 174 ret = -ENOMEM; 175 i--; 176 goto free_partial_kdata; 177 } 178 size *= sizeof(uint32_t); 179 if (copy_from_user(p->chunks[i].kdata, cdata, size)) { 180 ret = -EFAULT; 181 goto free_partial_kdata; 182 } 183 184 switch (p->chunks[i].chunk_id) { 185 case AMDGPU_CHUNK_ID_IB: 186 ++num_ibs; 187 break; 188 189 case AMDGPU_CHUNK_ID_FENCE: 190 size = sizeof(struct drm_amdgpu_cs_chunk_fence); 191 if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { 192 ret = -EINVAL; 193 goto free_partial_kdata; 194 } 195 196 ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata, 197 &uf_offset); 198 if (ret) 199 goto free_partial_kdata; 200 201 break; 202 203 case AMDGPU_CHUNK_ID_BO_HANDLES: 204 size = sizeof(struct drm_amdgpu_bo_list_in); 205 if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { 206 ret = -EINVAL; 207 goto free_partial_kdata; 208 } 209 210 ret = amdgpu_cs_bo_handles_chunk(p, p->chunks[i].kdata); 211 if (ret) 212 goto free_partial_kdata; 213 214 break; 215 216 case AMDGPU_CHUNK_ID_DEPENDENCIES: 217 case AMDGPU_CHUNK_ID_SYNCOBJ_IN: 218 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: 219 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: 220 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: 221 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: 222 break; 223 224 default: 225 ret = -EINVAL; 226 goto free_partial_kdata; 227 } 228 } 229 230 ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm); 231 if (ret) 232 goto free_all_kdata; 233 234 if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) { 235 ret = -ECANCELED; 236 goto free_all_kdata; 237 } 238 239 if (p->uf_entry.tv.bo) 240 p->job->uf_addr = uf_offset; 241 kvfree(chunk_array); 242 243 /* Use this opportunity to fill in task info for the vm */ 244 amdgpu_vm_set_task_info(vm); 245 246 return 0; 247 248 free_all_kdata: 249 i = p->nchunks - 1; 250 free_partial_kdata: 251 for (; i >= 0; i--) 252 kvfree(p->chunks[i].kdata); 253 kvfree(p->chunks); 254 p->chunks = NULL; 255 p->nchunks = 0; 256 free_chunk: 257 kvfree(chunk_array); 258 259 return ret; 260 } 261 262 /* Convert microseconds to bytes. */ 263 static u64 us_to_bytes(struct amdgpu_device *adev, s64 us) 264 { 265 if (us <= 0 || !adev->mm_stats.log2_max_MBps) 266 return 0; 267 268 /* Since accum_us is incremented by a million per second, just 269 * multiply it by the number of MB/s to get the number of bytes. 270 */ 271 return us << adev->mm_stats.log2_max_MBps; 272 } 273 274 static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes) 275 { 276 if (!adev->mm_stats.log2_max_MBps) 277 return 0; 278 279 return bytes >> adev->mm_stats.log2_max_MBps; 280 } 281 282 /* Returns how many bytes TTM can move right now. If no bytes can be moved, 283 * it returns 0. If it returns non-zero, it's OK to move at least one buffer, 284 * which means it can go over the threshold once. If that happens, the driver 285 * will be in debt and no other buffer migrations can be done until that debt 286 * is repaid. 287 * 288 * This approach allows moving a buffer of any size (it's important to allow 289 * that). 290 * 291 * The currency is simply time in microseconds and it increases as the clock 292 * ticks. The accumulated microseconds (us) are converted to bytes and 293 * returned. 294 */ 295 static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev, 296 u64 *max_bytes, 297 u64 *max_vis_bytes) 298 { 299 s64 time_us, increment_us; 300 u64 free_vram, total_vram, used_vram; 301 struct ttm_resource_manager *vram_man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 302 /* Allow a maximum of 200 accumulated ms. This is basically per-IB 303 * throttling. 304 * 305 * It means that in order to get full max MBps, at least 5 IBs per 306 * second must be submitted and not more than 200ms apart from each 307 * other. 308 */ 309 const s64 us_upper_bound = 200000; 310 311 if (!adev->mm_stats.log2_max_MBps) { 312 *max_bytes = 0; 313 *max_vis_bytes = 0; 314 return; 315 } 316 317 total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size); 318 used_vram = amdgpu_vram_mgr_usage(vram_man); 319 free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram; 320 321 spin_lock(&adev->mm_stats.lock); 322 323 /* Increase the amount of accumulated us. */ 324 time_us = ktime_to_us(ktime_get()); 325 increment_us = time_us - adev->mm_stats.last_update_us; 326 adev->mm_stats.last_update_us = time_us; 327 adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us, 328 us_upper_bound); 329 330 /* This prevents the short period of low performance when the VRAM 331 * usage is low and the driver is in debt or doesn't have enough 332 * accumulated us to fill VRAM quickly. 333 * 334 * The situation can occur in these cases: 335 * - a lot of VRAM is freed by userspace 336 * - the presence of a big buffer causes a lot of evictions 337 * (solution: split buffers into smaller ones) 338 * 339 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting 340 * accum_us to a positive number. 341 */ 342 if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) { 343 s64 min_us; 344 345 /* Be more aggresive on dGPUs. Try to fill a portion of free 346 * VRAM now. 347 */ 348 if (!(adev->flags & AMD_IS_APU)) 349 min_us = bytes_to_us(adev, free_vram / 4); 350 else 351 min_us = 0; /* Reset accum_us on APUs. */ 352 353 adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us); 354 } 355 356 /* This is set to 0 if the driver is in debt to disallow (optional) 357 * buffer moves. 358 */ 359 *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us); 360 361 /* Do the same for visible VRAM if half of it is free */ 362 if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) { 363 u64 total_vis_vram = adev->gmc.visible_vram_size; 364 u64 used_vis_vram = 365 amdgpu_vram_mgr_vis_usage(vram_man); 366 367 if (used_vis_vram < total_vis_vram) { 368 u64 free_vis_vram = total_vis_vram - used_vis_vram; 369 adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis + 370 increment_us, us_upper_bound); 371 372 if (free_vis_vram >= total_vis_vram / 2) 373 adev->mm_stats.accum_us_vis = 374 max(bytes_to_us(adev, free_vis_vram / 2), 375 adev->mm_stats.accum_us_vis); 376 } 377 378 *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis); 379 } else { 380 *max_vis_bytes = 0; 381 } 382 383 spin_unlock(&adev->mm_stats.lock); 384 } 385 386 /* Report how many bytes have really been moved for the last command 387 * submission. This can result in a debt that can stop buffer migrations 388 * temporarily. 389 */ 390 void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, 391 u64 num_vis_bytes) 392 { 393 spin_lock(&adev->mm_stats.lock); 394 adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes); 395 adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes); 396 spin_unlock(&adev->mm_stats.lock); 397 } 398 399 static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p, 400 struct amdgpu_bo *bo) 401 { 402 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 403 struct ttm_operation_ctx ctx = { 404 .interruptible = true, 405 .no_wait_gpu = false, 406 .resv = bo->tbo.base.resv 407 }; 408 uint32_t domain; 409 int r; 410 411 if (bo->tbo.pin_count) 412 return 0; 413 414 /* Don't move this buffer if we have depleted our allowance 415 * to move it. Don't move anything if the threshold is zero. 416 */ 417 if (p->bytes_moved < p->bytes_moved_threshold && 418 (!bo->tbo.base.dma_buf || 419 list_empty(&bo->tbo.base.dma_buf->attachments))) { 420 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 421 (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) { 422 /* And don't move a CPU_ACCESS_REQUIRED BO to limited 423 * visible VRAM if we've depleted our allowance to do 424 * that. 425 */ 426 if (p->bytes_moved_vis < p->bytes_moved_vis_threshold) 427 domain = bo->preferred_domains; 428 else 429 domain = bo->allowed_domains; 430 } else { 431 domain = bo->preferred_domains; 432 } 433 } else { 434 domain = bo->allowed_domains; 435 } 436 437 retry: 438 amdgpu_bo_placement_from_domain(bo, domain); 439 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 440 441 p->bytes_moved += ctx.bytes_moved; 442 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 443 amdgpu_bo_in_cpu_visible_vram(bo)) 444 p->bytes_moved_vis += ctx.bytes_moved; 445 446 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { 447 domain = bo->allowed_domains; 448 goto retry; 449 } 450 451 return r; 452 } 453 454 static int amdgpu_cs_validate(void *param, struct amdgpu_bo *bo) 455 { 456 struct amdgpu_cs_parser *p = param; 457 int r; 458 459 r = amdgpu_cs_bo_validate(p, bo); 460 if (r) 461 return r; 462 463 if (bo->shadow) 464 r = amdgpu_cs_bo_validate(p, bo->shadow); 465 466 return r; 467 } 468 469 static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, 470 struct list_head *validated) 471 { 472 struct ttm_operation_ctx ctx = { true, false }; 473 struct amdgpu_bo_list_entry *lobj; 474 int r; 475 476 list_for_each_entry(lobj, validated, tv.head) { 477 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(lobj->tv.bo); 478 struct mm_struct *usermm; 479 480 usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); 481 if (usermm && usermm != current->mm) 482 return -EPERM; 483 484 if (amdgpu_ttm_tt_is_userptr(bo->tbo.ttm) && 485 lobj->user_invalidated && lobj->user_pages) { 486 amdgpu_bo_placement_from_domain(bo, 487 AMDGPU_GEM_DOMAIN_CPU); 488 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 489 if (r) 490 return r; 491 492 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, 493 lobj->user_pages); 494 } 495 496 r = amdgpu_cs_validate(p, bo); 497 if (r) 498 return r; 499 500 kvfree(lobj->user_pages); 501 lobj->user_pages = NULL; 502 } 503 return 0; 504 } 505 506 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, 507 union drm_amdgpu_cs *cs) 508 { 509 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 510 struct amdgpu_vm *vm = &fpriv->vm; 511 struct amdgpu_bo_list_entry *e; 512 struct list_head duplicates; 513 struct amdgpu_bo *gds; 514 struct amdgpu_bo *gws; 515 struct amdgpu_bo *oa; 516 int r; 517 518 INIT_LIST_HEAD(&p->validated); 519 520 /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */ 521 if (cs->in.bo_list_handle) { 522 if (p->bo_list) 523 return -EINVAL; 524 525 r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle, 526 &p->bo_list); 527 if (r) 528 return r; 529 } else if (!p->bo_list) { 530 /* Create a empty bo_list when no handle is provided */ 531 r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0, 532 &p->bo_list); 533 if (r) 534 return r; 535 } 536 537 /* One for TTM and one for the CS job */ 538 amdgpu_bo_list_for_each_entry(e, p->bo_list) 539 e->tv.num_shared = 2; 540 541 amdgpu_bo_list_get_list(p->bo_list, &p->validated); 542 543 INIT_LIST_HEAD(&duplicates); 544 amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd); 545 546 if (p->uf_entry.tv.bo && !ttm_to_amdgpu_bo(p->uf_entry.tv.bo)->parent) 547 list_add(&p->uf_entry.tv.head, &p->validated); 548 549 /* Get userptr backing pages. If pages are updated after registered 550 * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do 551 * amdgpu_ttm_backend_bind() to flush and invalidate new pages 552 */ 553 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { 554 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); 555 bool userpage_invalidated = false; 556 int i; 557 558 e->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages, 559 sizeof(struct page *), 560 GFP_KERNEL | __GFP_ZERO); 561 if (!e->user_pages) { 562 DRM_ERROR("kvmalloc_array failure\n"); 563 return -ENOMEM; 564 } 565 566 r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages); 567 if (r) { 568 kvfree(e->user_pages); 569 e->user_pages = NULL; 570 return r; 571 } 572 573 for (i = 0; i < bo->tbo.ttm->num_pages; i++) { 574 if (bo->tbo.ttm->pages[i] != e->user_pages[i]) { 575 userpage_invalidated = true; 576 break; 577 } 578 } 579 e->user_invalidated = userpage_invalidated; 580 } 581 582 r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, 583 &duplicates); 584 if (unlikely(r != 0)) { 585 if (r != -ERESTARTSYS) 586 DRM_ERROR("ttm_eu_reserve_buffers failed.\n"); 587 goto out; 588 } 589 590 amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold, 591 &p->bytes_moved_vis_threshold); 592 p->bytes_moved = 0; 593 p->bytes_moved_vis = 0; 594 595 r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm, 596 amdgpu_cs_validate, p); 597 if (r) { 598 DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n"); 599 goto error_validate; 600 } 601 602 r = amdgpu_cs_list_validate(p, &duplicates); 603 if (r) 604 goto error_validate; 605 606 r = amdgpu_cs_list_validate(p, &p->validated); 607 if (r) 608 goto error_validate; 609 610 amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved, 611 p->bytes_moved_vis); 612 613 gds = p->bo_list->gds_obj; 614 gws = p->bo_list->gws_obj; 615 oa = p->bo_list->oa_obj; 616 617 amdgpu_bo_list_for_each_entry(e, p->bo_list) { 618 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); 619 620 /* Make sure we use the exclusive slot for shared BOs */ 621 if (bo->prime_shared_count) 622 e->tv.num_shared = 0; 623 e->bo_va = amdgpu_vm_bo_find(vm, bo); 624 } 625 626 if (gds) { 627 p->job->gds_base = amdgpu_bo_gpu_offset(gds) >> PAGE_SHIFT; 628 p->job->gds_size = amdgpu_bo_size(gds) >> PAGE_SHIFT; 629 } 630 if (gws) { 631 p->job->gws_base = amdgpu_bo_gpu_offset(gws) >> PAGE_SHIFT; 632 p->job->gws_size = amdgpu_bo_size(gws) >> PAGE_SHIFT; 633 } 634 if (oa) { 635 p->job->oa_base = amdgpu_bo_gpu_offset(oa) >> PAGE_SHIFT; 636 p->job->oa_size = amdgpu_bo_size(oa) >> PAGE_SHIFT; 637 } 638 639 if (!r && p->uf_entry.tv.bo) { 640 struct amdgpu_bo *uf = ttm_to_amdgpu_bo(p->uf_entry.tv.bo); 641 642 r = amdgpu_ttm_alloc_gart(&uf->tbo); 643 p->job->uf_addr += amdgpu_bo_gpu_offset(uf); 644 } 645 646 error_validate: 647 if (r) 648 ttm_eu_backoff_reservation(&p->ticket, &p->validated); 649 out: 650 return r; 651 } 652 653 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) 654 { 655 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 656 struct amdgpu_bo_list_entry *e; 657 int r; 658 659 list_for_each_entry(e, &p->validated, tv.head) { 660 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); 661 struct dma_resv *resv = bo->tbo.base.resv; 662 enum amdgpu_sync_mode sync_mode; 663 664 sync_mode = amdgpu_bo_explicit_sync(bo) ? 665 AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER; 666 r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, sync_mode, 667 &fpriv->vm); 668 if (r) 669 return r; 670 } 671 return 0; 672 } 673 674 /** 675 * cs_parser_fini() - clean parser states 676 * @parser: parser structure holding parsing context. 677 * @error: error number 678 * @backoff: indicator to backoff the reservation 679 * 680 * If error is set than unvalidate buffer, otherwise just free memory 681 * used by parsing context. 682 **/ 683 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, 684 bool backoff) 685 { 686 unsigned i; 687 688 if (error && backoff) 689 ttm_eu_backoff_reservation(&parser->ticket, 690 &parser->validated); 691 692 for (i = 0; i < parser->num_post_deps; i++) { 693 drm_syncobj_put(parser->post_deps[i].syncobj); 694 kfree(parser->post_deps[i].chain); 695 } 696 kfree(parser->post_deps); 697 698 dma_fence_put(parser->fence); 699 700 if (parser->ctx) { 701 mutex_unlock(&parser->ctx->lock); 702 amdgpu_ctx_put(parser->ctx); 703 } 704 if (parser->bo_list) 705 amdgpu_bo_list_put(parser->bo_list); 706 707 for (i = 0; i < parser->nchunks; i++) 708 kvfree(parser->chunks[i].kdata); 709 kvfree(parser->chunks); 710 if (parser->job) 711 amdgpu_job_free(parser->job); 712 if (parser->uf_entry.tv.bo) { 713 struct amdgpu_bo *uf = ttm_to_amdgpu_bo(parser->uf_entry.tv.bo); 714 715 amdgpu_bo_unref(&uf); 716 } 717 } 718 719 static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) 720 { 721 struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched); 722 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 723 struct amdgpu_device *adev = p->adev; 724 struct amdgpu_vm *vm = &fpriv->vm; 725 struct amdgpu_bo_list_entry *e; 726 struct amdgpu_bo_va *bo_va; 727 struct amdgpu_bo *bo; 728 int r; 729 730 /* Only for UVD/VCE VM emulation */ 731 if (ring->funcs->parse_cs || ring->funcs->patch_cs_in_place) { 732 unsigned i, j; 733 734 for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) { 735 struct drm_amdgpu_cs_chunk_ib *chunk_ib; 736 struct amdgpu_bo_va_mapping *m; 737 struct amdgpu_bo *aobj = NULL; 738 struct amdgpu_cs_chunk *chunk; 739 uint64_t offset, va_start; 740 struct amdgpu_ib *ib; 741 uint8_t *kptr; 742 743 chunk = &p->chunks[i]; 744 ib = &p->job->ibs[j]; 745 chunk_ib = chunk->kdata; 746 747 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) 748 continue; 749 750 va_start = chunk_ib->va_start & AMDGPU_GMC_HOLE_MASK; 751 r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m); 752 if (r) { 753 DRM_ERROR("IB va_start is invalid\n"); 754 return r; 755 } 756 757 if ((va_start + chunk_ib->ib_bytes) > 758 (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) { 759 DRM_ERROR("IB va_start+ib_bytes is invalid\n"); 760 return -EINVAL; 761 } 762 763 /* the IB should be reserved at this point */ 764 r = amdgpu_bo_kmap(aobj, (void **)&kptr); 765 if (r) { 766 return r; 767 } 768 769 offset = m->start * AMDGPU_GPU_PAGE_SIZE; 770 kptr += va_start - offset; 771 772 if (ring->funcs->parse_cs) { 773 memcpy(ib->ptr, kptr, chunk_ib->ib_bytes); 774 amdgpu_bo_kunmap(aobj); 775 776 r = amdgpu_ring_parse_cs(ring, p, j); 777 if (r) 778 return r; 779 } else { 780 ib->ptr = (uint32_t *)kptr; 781 r = amdgpu_ring_patch_cs_in_place(ring, p, j); 782 amdgpu_bo_kunmap(aobj); 783 if (r) 784 return r; 785 } 786 787 j++; 788 } 789 } 790 791 if (!p->job->vm) 792 return amdgpu_cs_sync_rings(p); 793 794 795 r = amdgpu_vm_clear_freed(adev, vm, NULL); 796 if (r) 797 return r; 798 799 r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); 800 if (r) 801 return r; 802 803 r = amdgpu_sync_vm_fence(&p->job->sync, fpriv->prt_va->last_pt_update); 804 if (r) 805 return r; 806 807 if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { 808 bo_va = fpriv->csa_va; 809 BUG_ON(!bo_va); 810 r = amdgpu_vm_bo_update(adev, bo_va, false); 811 if (r) 812 return r; 813 814 r = amdgpu_sync_vm_fence(&p->job->sync, bo_va->last_pt_update); 815 if (r) 816 return r; 817 } 818 819 amdgpu_bo_list_for_each_entry(e, p->bo_list) { 820 /* ignore duplicates */ 821 bo = ttm_to_amdgpu_bo(e->tv.bo); 822 if (!bo) 823 continue; 824 825 bo_va = e->bo_va; 826 if (bo_va == NULL) 827 continue; 828 829 r = amdgpu_vm_bo_update(adev, bo_va, false); 830 if (r) 831 return r; 832 833 r = amdgpu_sync_vm_fence(&p->job->sync, bo_va->last_pt_update); 834 if (r) 835 return r; 836 } 837 838 r = amdgpu_vm_handle_moved(adev, vm); 839 if (r) 840 return r; 841 842 r = amdgpu_vm_update_pdes(adev, vm, false); 843 if (r) 844 return r; 845 846 r = amdgpu_sync_vm_fence(&p->job->sync, vm->last_update); 847 if (r) 848 return r; 849 850 p->job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.base.bo); 851 852 if (amdgpu_vm_debug) { 853 /* Invalidate all BOs to test for userspace bugs */ 854 amdgpu_bo_list_for_each_entry(e, p->bo_list) { 855 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); 856 857 /* ignore duplicates */ 858 if (!bo) 859 continue; 860 861 amdgpu_vm_bo_invalidate(adev, bo, false); 862 } 863 } 864 865 return amdgpu_cs_sync_rings(p); 866 } 867 868 static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, 869 struct amdgpu_cs_parser *parser) 870 { 871 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 872 struct amdgpu_vm *vm = &fpriv->vm; 873 int r, ce_preempt = 0, de_preempt = 0; 874 struct amdgpu_ring *ring; 875 int i, j; 876 877 for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) { 878 struct amdgpu_cs_chunk *chunk; 879 struct amdgpu_ib *ib; 880 struct drm_amdgpu_cs_chunk_ib *chunk_ib; 881 struct drm_sched_entity *entity; 882 883 chunk = &parser->chunks[i]; 884 ib = &parser->job->ibs[j]; 885 chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata; 886 887 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) 888 continue; 889 890 if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && 891 (amdgpu_mcbp || amdgpu_sriov_vf(adev))) { 892 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) { 893 if (chunk_ib->flags & AMDGPU_IB_FLAG_CE) 894 ce_preempt++; 895 else 896 de_preempt++; 897 } 898 899 /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */ 900 if (ce_preempt > 1 || de_preempt > 1) 901 return -EINVAL; 902 } 903 904 r = amdgpu_ctx_get_entity(parser->ctx, chunk_ib->ip_type, 905 chunk_ib->ip_instance, chunk_ib->ring, 906 &entity); 907 if (r) 908 return r; 909 910 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE) 911 parser->job->preamble_status |= 912 AMDGPU_PREAMBLE_IB_PRESENT; 913 914 if (parser->entity && parser->entity != entity) 915 return -EINVAL; 916 917 /* Return if there is no run queue associated with this entity. 918 * Possibly because of disabled HW IP*/ 919 if (entity->rq == NULL) 920 return -EINVAL; 921 922 parser->entity = entity; 923 924 ring = to_amdgpu_ring(entity->rq->sched); 925 r = amdgpu_ib_get(adev, vm, ring->funcs->parse_cs ? 926 chunk_ib->ib_bytes : 0, 927 AMDGPU_IB_POOL_DELAYED, ib); 928 if (r) { 929 DRM_ERROR("Failed to get ib !\n"); 930 return r; 931 } 932 933 ib->gpu_addr = chunk_ib->va_start; 934 ib->length_dw = chunk_ib->ib_bytes / 4; 935 ib->flags = chunk_ib->flags; 936 937 j++; 938 } 939 940 /* MM engine doesn't support user fences */ 941 ring = to_amdgpu_ring(parser->entity->rq->sched); 942 if (parser->job->uf_addr && ring->funcs->no_user_fence) 943 return -EINVAL; 944 945 return amdgpu_ctx_wait_prev_fence(parser->ctx, parser->entity); 946 } 947 948 static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p, 949 struct amdgpu_cs_chunk *chunk) 950 { 951 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 952 unsigned num_deps; 953 int i, r; 954 struct drm_amdgpu_cs_chunk_dep *deps; 955 956 deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata; 957 num_deps = chunk->length_dw * 4 / 958 sizeof(struct drm_amdgpu_cs_chunk_dep); 959 960 for (i = 0; i < num_deps; ++i) { 961 struct amdgpu_ctx *ctx; 962 struct drm_sched_entity *entity; 963 struct dma_fence *fence; 964 965 ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id); 966 if (ctx == NULL) 967 return -EINVAL; 968 969 r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type, 970 deps[i].ip_instance, 971 deps[i].ring, &entity); 972 if (r) { 973 amdgpu_ctx_put(ctx); 974 return r; 975 } 976 977 fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle); 978 amdgpu_ctx_put(ctx); 979 980 if (IS_ERR(fence)) 981 return PTR_ERR(fence); 982 else if (!fence) 983 continue; 984 985 if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) { 986 struct drm_sched_fence *s_fence; 987 struct dma_fence *old = fence; 988 989 s_fence = to_drm_sched_fence(fence); 990 fence = dma_fence_get(&s_fence->scheduled); 991 dma_fence_put(old); 992 } 993 994 r = amdgpu_sync_fence(&p->job->sync, fence); 995 dma_fence_put(fence); 996 if (r) 997 return r; 998 } 999 return 0; 1000 } 1001 1002 static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p, 1003 uint32_t handle, u64 point, 1004 u64 flags) 1005 { 1006 struct dma_fence *fence; 1007 int r; 1008 1009 r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence); 1010 if (r) { 1011 DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n", 1012 handle, point, r); 1013 return r; 1014 } 1015 1016 r = amdgpu_sync_fence(&p->job->sync, fence); 1017 dma_fence_put(fence); 1018 1019 return r; 1020 } 1021 1022 static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser *p, 1023 struct amdgpu_cs_chunk *chunk) 1024 { 1025 struct drm_amdgpu_cs_chunk_sem *deps; 1026 unsigned num_deps; 1027 int i, r; 1028 1029 deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; 1030 num_deps = chunk->length_dw * 4 / 1031 sizeof(struct drm_amdgpu_cs_chunk_sem); 1032 for (i = 0; i < num_deps; ++i) { 1033 r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle, 1034 0, 0); 1035 if (r) 1036 return r; 1037 } 1038 1039 return 0; 1040 } 1041 1042 1043 static int amdgpu_cs_process_syncobj_timeline_in_dep(struct amdgpu_cs_parser *p, 1044 struct amdgpu_cs_chunk *chunk) 1045 { 1046 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps; 1047 unsigned num_deps; 1048 int i, r; 1049 1050 syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata; 1051 num_deps = chunk->length_dw * 4 / 1052 sizeof(struct drm_amdgpu_cs_chunk_syncobj); 1053 for (i = 0; i < num_deps; ++i) { 1054 r = amdgpu_syncobj_lookup_and_add_to_sync(p, 1055 syncobj_deps[i].handle, 1056 syncobj_deps[i].point, 1057 syncobj_deps[i].flags); 1058 if (r) 1059 return r; 1060 } 1061 1062 return 0; 1063 } 1064 1065 static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p, 1066 struct amdgpu_cs_chunk *chunk) 1067 { 1068 struct drm_amdgpu_cs_chunk_sem *deps; 1069 unsigned num_deps; 1070 int i; 1071 1072 deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; 1073 num_deps = chunk->length_dw * 4 / 1074 sizeof(struct drm_amdgpu_cs_chunk_sem); 1075 1076 if (p->post_deps) 1077 return -EINVAL; 1078 1079 p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), 1080 GFP_KERNEL); 1081 p->num_post_deps = 0; 1082 1083 if (!p->post_deps) 1084 return -ENOMEM; 1085 1086 1087 for (i = 0; i < num_deps; ++i) { 1088 p->post_deps[i].syncobj = 1089 drm_syncobj_find(p->filp, deps[i].handle); 1090 if (!p->post_deps[i].syncobj) 1091 return -EINVAL; 1092 p->post_deps[i].chain = NULL; 1093 p->post_deps[i].point = 0; 1094 p->num_post_deps++; 1095 } 1096 1097 return 0; 1098 } 1099 1100 1101 static int amdgpu_cs_process_syncobj_timeline_out_dep(struct amdgpu_cs_parser *p, 1102 struct amdgpu_cs_chunk *chunk) 1103 { 1104 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps; 1105 unsigned num_deps; 1106 int i; 1107 1108 syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata; 1109 num_deps = chunk->length_dw * 4 / 1110 sizeof(struct drm_amdgpu_cs_chunk_syncobj); 1111 1112 if (p->post_deps) 1113 return -EINVAL; 1114 1115 p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), 1116 GFP_KERNEL); 1117 p->num_post_deps = 0; 1118 1119 if (!p->post_deps) 1120 return -ENOMEM; 1121 1122 for (i = 0; i < num_deps; ++i) { 1123 struct amdgpu_cs_post_dep *dep = &p->post_deps[i]; 1124 1125 dep->chain = NULL; 1126 if (syncobj_deps[i].point) { 1127 dep->chain = kmalloc(sizeof(*dep->chain), GFP_KERNEL); 1128 if (!dep->chain) 1129 return -ENOMEM; 1130 } 1131 1132 dep->syncobj = drm_syncobj_find(p->filp, 1133 syncobj_deps[i].handle); 1134 if (!dep->syncobj) { 1135 kfree(dep->chain); 1136 return -EINVAL; 1137 } 1138 dep->point = syncobj_deps[i].point; 1139 p->num_post_deps++; 1140 } 1141 1142 return 0; 1143 } 1144 1145 static int amdgpu_cs_dependencies(struct amdgpu_device *adev, 1146 struct amdgpu_cs_parser *p) 1147 { 1148 int i, r; 1149 1150 for (i = 0; i < p->nchunks; ++i) { 1151 struct amdgpu_cs_chunk *chunk; 1152 1153 chunk = &p->chunks[i]; 1154 1155 switch (chunk->chunk_id) { 1156 case AMDGPU_CHUNK_ID_DEPENDENCIES: 1157 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: 1158 r = amdgpu_cs_process_fence_dep(p, chunk); 1159 if (r) 1160 return r; 1161 break; 1162 case AMDGPU_CHUNK_ID_SYNCOBJ_IN: 1163 r = amdgpu_cs_process_syncobj_in_dep(p, chunk); 1164 if (r) 1165 return r; 1166 break; 1167 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: 1168 r = amdgpu_cs_process_syncobj_out_dep(p, chunk); 1169 if (r) 1170 return r; 1171 break; 1172 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: 1173 r = amdgpu_cs_process_syncobj_timeline_in_dep(p, chunk); 1174 if (r) 1175 return r; 1176 break; 1177 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: 1178 r = amdgpu_cs_process_syncobj_timeline_out_dep(p, chunk); 1179 if (r) 1180 return r; 1181 break; 1182 } 1183 } 1184 1185 return 0; 1186 } 1187 1188 static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p) 1189 { 1190 int i; 1191 1192 for (i = 0; i < p->num_post_deps; ++i) { 1193 if (p->post_deps[i].chain && p->post_deps[i].point) { 1194 drm_syncobj_add_point(p->post_deps[i].syncobj, 1195 p->post_deps[i].chain, 1196 p->fence, p->post_deps[i].point); 1197 p->post_deps[i].chain = NULL; 1198 } else { 1199 drm_syncobj_replace_fence(p->post_deps[i].syncobj, 1200 p->fence); 1201 } 1202 } 1203 } 1204 1205 static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, 1206 union drm_amdgpu_cs *cs) 1207 { 1208 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 1209 struct drm_sched_entity *entity = p->entity; 1210 struct amdgpu_bo_list_entry *e; 1211 struct amdgpu_job *job; 1212 uint64_t seq; 1213 int r; 1214 1215 job = p->job; 1216 p->job = NULL; 1217 1218 r = drm_sched_job_init(&job->base, entity, &fpriv->vm); 1219 if (r) 1220 goto error_unlock; 1221 1222 /* No memory allocation is allowed while holding the notifier lock. 1223 * The lock is held until amdgpu_cs_submit is finished and fence is 1224 * added to BOs. 1225 */ 1226 mutex_lock(&p->adev->notifier_lock); 1227 1228 /* If userptr are invalidated after amdgpu_cs_parser_bos(), return 1229 * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl. 1230 */ 1231 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { 1232 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); 1233 1234 r |= !amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); 1235 } 1236 if (r) { 1237 r = -EAGAIN; 1238 goto error_abort; 1239 } 1240 1241 p->fence = dma_fence_get(&job->base.s_fence->finished); 1242 1243 amdgpu_ctx_add_fence(p->ctx, entity, p->fence, &seq); 1244 amdgpu_cs_post_dependencies(p); 1245 1246 if ((job->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) && 1247 !p->ctx->preamble_presented) { 1248 job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST; 1249 p->ctx->preamble_presented = true; 1250 } 1251 1252 cs->out.handle = seq; 1253 job->uf_sequence = seq; 1254 1255 amdgpu_job_free_resources(job); 1256 1257 trace_amdgpu_cs_ioctl(job); 1258 amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->ticket); 1259 drm_sched_entity_push_job(&job->base, entity); 1260 1261 amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm); 1262 1263 ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence); 1264 mutex_unlock(&p->adev->notifier_lock); 1265 1266 return 0; 1267 1268 error_abort: 1269 drm_sched_job_cleanup(&job->base); 1270 mutex_unlock(&p->adev->notifier_lock); 1271 1272 error_unlock: 1273 amdgpu_job_free(job); 1274 return r; 1275 } 1276 1277 static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *parser) 1278 { 1279 int i; 1280 1281 if (!trace_amdgpu_cs_enabled()) 1282 return; 1283 1284 for (i = 0; i < parser->job->num_ibs; i++) 1285 trace_amdgpu_cs(parser, i); 1286 } 1287 1288 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 1289 { 1290 struct amdgpu_device *adev = drm_to_adev(dev); 1291 union drm_amdgpu_cs *cs = data; 1292 struct amdgpu_cs_parser parser = {}; 1293 bool reserved_buffers = false; 1294 int r; 1295 1296 if (amdgpu_ras_intr_triggered()) 1297 return -EHWPOISON; 1298 1299 if (!adev->accel_working) 1300 return -EBUSY; 1301 1302 parser.adev = adev; 1303 parser.filp = filp; 1304 1305 r = amdgpu_cs_parser_init(&parser, data); 1306 if (r) { 1307 if (printk_ratelimit()) 1308 DRM_ERROR("Failed to initialize parser %d!\n", r); 1309 goto out; 1310 } 1311 1312 r = amdgpu_cs_ib_fill(adev, &parser); 1313 if (r) 1314 goto out; 1315 1316 r = amdgpu_cs_dependencies(adev, &parser); 1317 if (r) { 1318 DRM_ERROR("Failed in the dependencies handling %d!\n", r); 1319 goto out; 1320 } 1321 1322 r = amdgpu_cs_parser_bos(&parser, data); 1323 if (r) { 1324 if (r == -ENOMEM) 1325 DRM_ERROR("Not enough memory for command submission!\n"); 1326 else if (r != -ERESTARTSYS && r != -EAGAIN) 1327 DRM_ERROR("Failed to process the buffer list %d!\n", r); 1328 goto out; 1329 } 1330 1331 reserved_buffers = true; 1332 1333 trace_amdgpu_cs_ibs(&parser); 1334 1335 r = amdgpu_cs_vm_handling(&parser); 1336 if (r) 1337 goto out; 1338 1339 r = amdgpu_cs_submit(&parser, cs); 1340 1341 out: 1342 amdgpu_cs_parser_fini(&parser, r, reserved_buffers); 1343 1344 return r; 1345 } 1346 1347 /** 1348 * amdgpu_cs_wait_ioctl - wait for a command submission to finish 1349 * 1350 * @dev: drm device 1351 * @data: data from userspace 1352 * @filp: file private 1353 * 1354 * Wait for the command submission identified by handle to finish. 1355 */ 1356 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, 1357 struct drm_file *filp) 1358 { 1359 union drm_amdgpu_wait_cs *wait = data; 1360 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout); 1361 struct drm_sched_entity *entity; 1362 struct amdgpu_ctx *ctx; 1363 struct dma_fence *fence; 1364 long r; 1365 1366 ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id); 1367 if (ctx == NULL) 1368 return -EINVAL; 1369 1370 r = amdgpu_ctx_get_entity(ctx, wait->in.ip_type, wait->in.ip_instance, 1371 wait->in.ring, &entity); 1372 if (r) { 1373 amdgpu_ctx_put(ctx); 1374 return r; 1375 } 1376 1377 fence = amdgpu_ctx_get_fence(ctx, entity, wait->in.handle); 1378 if (IS_ERR(fence)) 1379 r = PTR_ERR(fence); 1380 else if (fence) { 1381 r = dma_fence_wait_timeout(fence, true, timeout); 1382 if (r > 0 && fence->error) 1383 r = fence->error; 1384 dma_fence_put(fence); 1385 } else 1386 r = 1; 1387 1388 amdgpu_ctx_put(ctx); 1389 if (r < 0) 1390 return r; 1391 1392 memset(wait, 0, sizeof(*wait)); 1393 wait->out.status = (r == 0); 1394 1395 return 0; 1396 } 1397 1398 /** 1399 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence 1400 * 1401 * @adev: amdgpu device 1402 * @filp: file private 1403 * @user: drm_amdgpu_fence copied from user space 1404 */ 1405 static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev, 1406 struct drm_file *filp, 1407 struct drm_amdgpu_fence *user) 1408 { 1409 struct drm_sched_entity *entity; 1410 struct amdgpu_ctx *ctx; 1411 struct dma_fence *fence; 1412 int r; 1413 1414 ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id); 1415 if (ctx == NULL) 1416 return ERR_PTR(-EINVAL); 1417 1418 r = amdgpu_ctx_get_entity(ctx, user->ip_type, user->ip_instance, 1419 user->ring, &entity); 1420 if (r) { 1421 amdgpu_ctx_put(ctx); 1422 return ERR_PTR(r); 1423 } 1424 1425 fence = amdgpu_ctx_get_fence(ctx, entity, user->seq_no); 1426 amdgpu_ctx_put(ctx); 1427 1428 return fence; 1429 } 1430 1431 int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data, 1432 struct drm_file *filp) 1433 { 1434 struct amdgpu_device *adev = drm_to_adev(dev); 1435 union drm_amdgpu_fence_to_handle *info = data; 1436 struct dma_fence *fence; 1437 struct drm_syncobj *syncobj; 1438 struct sync_file *sync_file; 1439 int fd, r; 1440 1441 fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence); 1442 if (IS_ERR(fence)) 1443 return PTR_ERR(fence); 1444 1445 if (!fence) 1446 fence = dma_fence_get_stub(); 1447 1448 switch (info->in.what) { 1449 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ: 1450 r = drm_syncobj_create(&syncobj, 0, fence); 1451 dma_fence_put(fence); 1452 if (r) 1453 return r; 1454 r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle); 1455 drm_syncobj_put(syncobj); 1456 return r; 1457 1458 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD: 1459 r = drm_syncobj_create(&syncobj, 0, fence); 1460 dma_fence_put(fence); 1461 if (r) 1462 return r; 1463 r = drm_syncobj_get_fd(syncobj, (int *)&info->out.handle); 1464 drm_syncobj_put(syncobj); 1465 return r; 1466 1467 case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD: 1468 fd = get_unused_fd_flags(O_CLOEXEC); 1469 if (fd < 0) { 1470 dma_fence_put(fence); 1471 return fd; 1472 } 1473 1474 sync_file = sync_file_create(fence); 1475 dma_fence_put(fence); 1476 if (!sync_file) { 1477 put_unused_fd(fd); 1478 return -ENOMEM; 1479 } 1480 1481 fd_install(fd, sync_file->file); 1482 info->out.handle = fd; 1483 return 0; 1484 1485 default: 1486 return -EINVAL; 1487 } 1488 } 1489 1490 /** 1491 * amdgpu_cs_wait_all_fence - wait on all fences to signal 1492 * 1493 * @adev: amdgpu device 1494 * @filp: file private 1495 * @wait: wait parameters 1496 * @fences: array of drm_amdgpu_fence 1497 */ 1498 static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev, 1499 struct drm_file *filp, 1500 union drm_amdgpu_wait_fences *wait, 1501 struct drm_amdgpu_fence *fences) 1502 { 1503 uint32_t fence_count = wait->in.fence_count; 1504 unsigned int i; 1505 long r = 1; 1506 1507 for (i = 0; i < fence_count; i++) { 1508 struct dma_fence *fence; 1509 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); 1510 1511 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); 1512 if (IS_ERR(fence)) 1513 return PTR_ERR(fence); 1514 else if (!fence) 1515 continue; 1516 1517 r = dma_fence_wait_timeout(fence, true, timeout); 1518 dma_fence_put(fence); 1519 if (r < 0) 1520 return r; 1521 1522 if (r == 0) 1523 break; 1524 1525 if (fence->error) 1526 return fence->error; 1527 } 1528 1529 memset(wait, 0, sizeof(*wait)); 1530 wait->out.status = (r > 0); 1531 1532 return 0; 1533 } 1534 1535 /** 1536 * amdgpu_cs_wait_any_fence - wait on any fence to signal 1537 * 1538 * @adev: amdgpu device 1539 * @filp: file private 1540 * @wait: wait parameters 1541 * @fences: array of drm_amdgpu_fence 1542 */ 1543 static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev, 1544 struct drm_file *filp, 1545 union drm_amdgpu_wait_fences *wait, 1546 struct drm_amdgpu_fence *fences) 1547 { 1548 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); 1549 uint32_t fence_count = wait->in.fence_count; 1550 uint32_t first = ~0; 1551 struct dma_fence **array; 1552 unsigned int i; 1553 long r; 1554 1555 /* Prepare the fence array */ 1556 array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL); 1557 1558 if (array == NULL) 1559 return -ENOMEM; 1560 1561 for (i = 0; i < fence_count; i++) { 1562 struct dma_fence *fence; 1563 1564 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); 1565 if (IS_ERR(fence)) { 1566 r = PTR_ERR(fence); 1567 goto err_free_fence_array; 1568 } else if (fence) { 1569 array[i] = fence; 1570 } else { /* NULL, the fence has been already signaled */ 1571 r = 1; 1572 first = i; 1573 goto out; 1574 } 1575 } 1576 1577 r = dma_fence_wait_any_timeout(array, fence_count, true, timeout, 1578 &first); 1579 if (r < 0) 1580 goto err_free_fence_array; 1581 1582 out: 1583 memset(wait, 0, sizeof(*wait)); 1584 wait->out.status = (r > 0); 1585 wait->out.first_signaled = first; 1586 1587 if (first < fence_count && array[first]) 1588 r = array[first]->error; 1589 else 1590 r = 0; 1591 1592 err_free_fence_array: 1593 for (i = 0; i < fence_count; i++) 1594 dma_fence_put(array[i]); 1595 kfree(array); 1596 1597 return r; 1598 } 1599 1600 /** 1601 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish 1602 * 1603 * @dev: drm device 1604 * @data: data from userspace 1605 * @filp: file private 1606 */ 1607 int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data, 1608 struct drm_file *filp) 1609 { 1610 struct amdgpu_device *adev = drm_to_adev(dev); 1611 union drm_amdgpu_wait_fences *wait = data; 1612 uint32_t fence_count = wait->in.fence_count; 1613 struct drm_amdgpu_fence *fences_user; 1614 struct drm_amdgpu_fence *fences; 1615 int r; 1616 1617 /* Get the fences from userspace */ 1618 fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence), 1619 GFP_KERNEL); 1620 if (fences == NULL) 1621 return -ENOMEM; 1622 1623 fences_user = u64_to_user_ptr(wait->in.fences); 1624 if (copy_from_user(fences, fences_user, 1625 sizeof(struct drm_amdgpu_fence) * fence_count)) { 1626 r = -EFAULT; 1627 goto err_free_fences; 1628 } 1629 1630 if (wait->in.wait_all) 1631 r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences); 1632 else 1633 r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences); 1634 1635 err_free_fences: 1636 kfree(fences); 1637 1638 return r; 1639 } 1640 1641 /** 1642 * amdgpu_cs_find_bo_va - find bo_va for VM address 1643 * 1644 * @parser: command submission parser context 1645 * @addr: VM address 1646 * @bo: resulting BO of the mapping found 1647 * @map: Placeholder to return found BO mapping 1648 * 1649 * Search the buffer objects in the command submission context for a certain 1650 * virtual memory address. Returns allocation structure when found, NULL 1651 * otherwise. 1652 */ 1653 int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser, 1654 uint64_t addr, struct amdgpu_bo **bo, 1655 struct amdgpu_bo_va_mapping **map) 1656 { 1657 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 1658 struct ttm_operation_ctx ctx = { false, false }; 1659 struct amdgpu_vm *vm = &fpriv->vm; 1660 struct amdgpu_bo_va_mapping *mapping; 1661 int r; 1662 1663 addr /= AMDGPU_GPU_PAGE_SIZE; 1664 1665 mapping = amdgpu_vm_bo_lookup_mapping(vm, addr); 1666 if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo) 1667 return -EINVAL; 1668 1669 *bo = mapping->bo_va->base.bo; 1670 *map = mapping; 1671 1672 /* Double check that the BO is reserved by this CS */ 1673 if (dma_resv_locking_ctx((*bo)->tbo.base.resv) != &parser->ticket) 1674 return -EINVAL; 1675 1676 if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) { 1677 (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 1678 amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains); 1679 r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx); 1680 if (r) 1681 return r; 1682 } 1683 1684 return amdgpu_ttm_alloc_gart(&(*bo)->tbo); 1685 } 1686