1 /* 2 * Copyright 2008 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 22 * DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: 25 * Jerome Glisse <glisse@freedesktop.org> 26 */ 27 #include <linux/pagemap.h> 28 #include <linux/sync_file.h> 29 #include <drm/drmP.h> 30 #include <drm/amdgpu_drm.h> 31 #include <drm/drm_syncobj.h> 32 #include "amdgpu.h" 33 #include "amdgpu_trace.h" 34 35 static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p, 36 struct drm_amdgpu_cs_chunk_fence *data, 37 uint32_t *offset) 38 { 39 struct drm_gem_object *gobj; 40 unsigned long size; 41 42 gobj = drm_gem_object_lookup(p->filp, data->handle); 43 if (gobj == NULL) 44 return -EINVAL; 45 46 p->uf_entry.robj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); 47 p->uf_entry.priority = 0; 48 p->uf_entry.tv.bo = &p->uf_entry.robj->tbo; 49 p->uf_entry.tv.shared = true; 50 p->uf_entry.user_pages = NULL; 51 52 size = amdgpu_bo_size(p->uf_entry.robj); 53 if (size != PAGE_SIZE || (data->offset + 8) > size) 54 return -EINVAL; 55 56 *offset = data->offset; 57 58 drm_gem_object_put_unlocked(gobj); 59 60 if (amdgpu_ttm_tt_get_usermm(p->uf_entry.robj->tbo.ttm)) { 61 amdgpu_bo_unref(&p->uf_entry.robj); 62 return -EINVAL; 63 } 64 65 return 0; 66 } 67 68 static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data) 69 { 70 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 71 struct amdgpu_vm *vm = &fpriv->vm; 72 union drm_amdgpu_cs *cs = data; 73 uint64_t *chunk_array_user; 74 uint64_t *chunk_array; 75 unsigned size, num_ibs = 0; 76 uint32_t uf_offset = 0; 77 int i; 78 int ret; 79 80 if (cs->in.num_chunks == 0) 81 return 0; 82 83 chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); 84 if (!chunk_array) 85 return -ENOMEM; 86 87 p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); 88 if (!p->ctx) { 89 ret = -EINVAL; 90 goto free_chunk; 91 } 92 93 mutex_lock(&p->ctx->lock); 94 95 /* get chunks */ 96 chunk_array_user = u64_to_user_ptr(cs->in.chunks); 97 if (copy_from_user(chunk_array, chunk_array_user, 98 sizeof(uint64_t)*cs->in.num_chunks)) { 99 ret = -EFAULT; 100 goto free_chunk; 101 } 102 103 p->nchunks = cs->in.num_chunks; 104 p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk), 105 GFP_KERNEL); 106 if (!p->chunks) { 107 ret = -ENOMEM; 108 goto free_chunk; 109 } 110 111 for (i = 0; i < p->nchunks; i++) { 112 struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL; 113 struct drm_amdgpu_cs_chunk user_chunk; 114 uint32_t __user *cdata; 115 116 chunk_ptr = u64_to_user_ptr(chunk_array[i]); 117 if (copy_from_user(&user_chunk, chunk_ptr, 118 sizeof(struct drm_amdgpu_cs_chunk))) { 119 ret = -EFAULT; 120 i--; 121 goto free_partial_kdata; 122 } 123 p->chunks[i].chunk_id = user_chunk.chunk_id; 124 p->chunks[i].length_dw = user_chunk.length_dw; 125 126 size = p->chunks[i].length_dw; 127 cdata = u64_to_user_ptr(user_chunk.chunk_data); 128 129 p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL); 130 if (p->chunks[i].kdata == NULL) { 131 ret = -ENOMEM; 132 i--; 133 goto free_partial_kdata; 134 } 135 size *= sizeof(uint32_t); 136 if (copy_from_user(p->chunks[i].kdata, cdata, size)) { 137 ret = -EFAULT; 138 goto free_partial_kdata; 139 } 140 141 switch (p->chunks[i].chunk_id) { 142 case AMDGPU_CHUNK_ID_IB: 143 ++num_ibs; 144 break; 145 146 case AMDGPU_CHUNK_ID_FENCE: 147 size = sizeof(struct drm_amdgpu_cs_chunk_fence); 148 if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { 149 ret = -EINVAL; 150 goto free_partial_kdata; 151 } 152 153 ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata, 154 &uf_offset); 155 if (ret) 156 goto free_partial_kdata; 157 158 break; 159 160 case AMDGPU_CHUNK_ID_DEPENDENCIES: 161 case AMDGPU_CHUNK_ID_SYNCOBJ_IN: 162 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: 163 break; 164 165 default: 166 ret = -EINVAL; 167 goto free_partial_kdata; 168 } 169 } 170 171 ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm); 172 if (ret) 173 goto free_all_kdata; 174 175 if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) { 176 ret = -ECANCELED; 177 goto free_all_kdata; 178 } 179 180 if (p->uf_entry.robj) 181 p->job->uf_addr = uf_offset; 182 kfree(chunk_array); 183 return 0; 184 185 free_all_kdata: 186 i = p->nchunks - 1; 187 free_partial_kdata: 188 for (; i >= 0; i--) 189 kvfree(p->chunks[i].kdata); 190 kfree(p->chunks); 191 p->chunks = NULL; 192 p->nchunks = 0; 193 free_chunk: 194 kfree(chunk_array); 195 196 return ret; 197 } 198 199 /* Convert microseconds to bytes. */ 200 static u64 us_to_bytes(struct amdgpu_device *adev, s64 us) 201 { 202 if (us <= 0 || !adev->mm_stats.log2_max_MBps) 203 return 0; 204 205 /* Since accum_us is incremented by a million per second, just 206 * multiply it by the number of MB/s to get the number of bytes. 207 */ 208 return us << adev->mm_stats.log2_max_MBps; 209 } 210 211 static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes) 212 { 213 if (!adev->mm_stats.log2_max_MBps) 214 return 0; 215 216 return bytes >> adev->mm_stats.log2_max_MBps; 217 } 218 219 /* Returns how many bytes TTM can move right now. If no bytes can be moved, 220 * it returns 0. If it returns non-zero, it's OK to move at least one buffer, 221 * which means it can go over the threshold once. If that happens, the driver 222 * will be in debt and no other buffer migrations can be done until that debt 223 * is repaid. 224 * 225 * This approach allows moving a buffer of any size (it's important to allow 226 * that). 227 * 228 * The currency is simply time in microseconds and it increases as the clock 229 * ticks. The accumulated microseconds (us) are converted to bytes and 230 * returned. 231 */ 232 static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev, 233 u64 *max_bytes, 234 u64 *max_vis_bytes) 235 { 236 s64 time_us, increment_us; 237 u64 free_vram, total_vram, used_vram; 238 239 /* Allow a maximum of 200 accumulated ms. This is basically per-IB 240 * throttling. 241 * 242 * It means that in order to get full max MBps, at least 5 IBs per 243 * second must be submitted and not more than 200ms apart from each 244 * other. 245 */ 246 const s64 us_upper_bound = 200000; 247 248 if (!adev->mm_stats.log2_max_MBps) { 249 *max_bytes = 0; 250 *max_vis_bytes = 0; 251 return; 252 } 253 254 total_vram = adev->mc.real_vram_size - adev->vram_pin_size; 255 used_vram = amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); 256 free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram; 257 258 spin_lock(&adev->mm_stats.lock); 259 260 /* Increase the amount of accumulated us. */ 261 time_us = ktime_to_us(ktime_get()); 262 increment_us = time_us - adev->mm_stats.last_update_us; 263 adev->mm_stats.last_update_us = time_us; 264 adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us, 265 us_upper_bound); 266 267 /* This prevents the short period of low performance when the VRAM 268 * usage is low and the driver is in debt or doesn't have enough 269 * accumulated us to fill VRAM quickly. 270 * 271 * The situation can occur in these cases: 272 * - a lot of VRAM is freed by userspace 273 * - the presence of a big buffer causes a lot of evictions 274 * (solution: split buffers into smaller ones) 275 * 276 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting 277 * accum_us to a positive number. 278 */ 279 if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) { 280 s64 min_us; 281 282 /* Be more aggresive on dGPUs. Try to fill a portion of free 283 * VRAM now. 284 */ 285 if (!(adev->flags & AMD_IS_APU)) 286 min_us = bytes_to_us(adev, free_vram / 4); 287 else 288 min_us = 0; /* Reset accum_us on APUs. */ 289 290 adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us); 291 } 292 293 /* This is set to 0 if the driver is in debt to disallow (optional) 294 * buffer moves. 295 */ 296 *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us); 297 298 /* Do the same for visible VRAM if half of it is free */ 299 if (adev->mc.visible_vram_size < adev->mc.real_vram_size) { 300 u64 total_vis_vram = adev->mc.visible_vram_size; 301 u64 used_vis_vram = 302 amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]); 303 304 if (used_vis_vram < total_vis_vram) { 305 u64 free_vis_vram = total_vis_vram - used_vis_vram; 306 adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis + 307 increment_us, us_upper_bound); 308 309 if (free_vis_vram >= total_vis_vram / 2) 310 adev->mm_stats.accum_us_vis = 311 max(bytes_to_us(adev, free_vis_vram / 2), 312 adev->mm_stats.accum_us_vis); 313 } 314 315 *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis); 316 } else { 317 *max_vis_bytes = 0; 318 } 319 320 spin_unlock(&adev->mm_stats.lock); 321 } 322 323 /* Report how many bytes have really been moved for the last command 324 * submission. This can result in a debt that can stop buffer migrations 325 * temporarily. 326 */ 327 void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, 328 u64 num_vis_bytes) 329 { 330 spin_lock(&adev->mm_stats.lock); 331 adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes); 332 adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes); 333 spin_unlock(&adev->mm_stats.lock); 334 } 335 336 static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p, 337 struct amdgpu_bo *bo) 338 { 339 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 340 u64 initial_bytes_moved, bytes_moved; 341 uint32_t domain; 342 int r; 343 344 if (bo->pin_count) 345 return 0; 346 347 /* Don't move this buffer if we have depleted our allowance 348 * to move it. Don't move anything if the threshold is zero. 349 */ 350 if (p->bytes_moved < p->bytes_moved_threshold) { 351 if (adev->mc.visible_vram_size < adev->mc.real_vram_size && 352 (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) { 353 /* And don't move a CPU_ACCESS_REQUIRED BO to limited 354 * visible VRAM if we've depleted our allowance to do 355 * that. 356 */ 357 if (p->bytes_moved_vis < p->bytes_moved_vis_threshold) 358 domain = bo->preferred_domains; 359 else 360 domain = bo->allowed_domains; 361 } else { 362 domain = bo->preferred_domains; 363 } 364 } else { 365 domain = bo->allowed_domains; 366 } 367 368 retry: 369 amdgpu_ttm_placement_from_domain(bo, domain); 370 initial_bytes_moved = atomic64_read(&adev->num_bytes_moved); 371 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 372 bytes_moved = atomic64_read(&adev->num_bytes_moved) - 373 initial_bytes_moved; 374 p->bytes_moved += bytes_moved; 375 if (adev->mc.visible_vram_size < adev->mc.real_vram_size && 376 bo->tbo.mem.mem_type == TTM_PL_VRAM && 377 bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT) 378 p->bytes_moved_vis += bytes_moved; 379 380 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { 381 domain = bo->allowed_domains; 382 goto retry; 383 } 384 385 return r; 386 } 387 388 /* Last resort, try to evict something from the current working set */ 389 static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p, 390 struct amdgpu_bo *validated) 391 { 392 uint32_t domain = validated->allowed_domains; 393 int r; 394 395 if (!p->evictable) 396 return false; 397 398 for (;&p->evictable->tv.head != &p->validated; 399 p->evictable = list_prev_entry(p->evictable, tv.head)) { 400 401 struct amdgpu_bo_list_entry *candidate = p->evictable; 402 struct amdgpu_bo *bo = candidate->robj; 403 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 404 u64 initial_bytes_moved, bytes_moved; 405 bool update_bytes_moved_vis; 406 uint32_t other; 407 408 /* If we reached our current BO we can forget it */ 409 if (candidate->robj == validated) 410 break; 411 412 other = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); 413 414 /* Check if this BO is in one of the domains we need space for */ 415 if (!(other & domain)) 416 continue; 417 418 /* Check if we can move this BO somewhere else */ 419 other = bo->allowed_domains & ~domain; 420 if (!other) 421 continue; 422 423 /* Good we can try to move this BO somewhere else */ 424 amdgpu_ttm_placement_from_domain(bo, other); 425 update_bytes_moved_vis = 426 adev->mc.visible_vram_size < adev->mc.real_vram_size && 427 bo->tbo.mem.mem_type == TTM_PL_VRAM && 428 bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT; 429 initial_bytes_moved = atomic64_read(&adev->num_bytes_moved); 430 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 431 bytes_moved = atomic64_read(&adev->num_bytes_moved) - 432 initial_bytes_moved; 433 p->bytes_moved += bytes_moved; 434 if (update_bytes_moved_vis) 435 p->bytes_moved_vis += bytes_moved; 436 437 if (unlikely(r)) 438 break; 439 440 p->evictable = list_prev_entry(p->evictable, tv.head); 441 list_move(&candidate->tv.head, &p->validated); 442 443 return true; 444 } 445 446 return false; 447 } 448 449 static int amdgpu_cs_validate(void *param, struct amdgpu_bo *bo) 450 { 451 struct amdgpu_cs_parser *p = param; 452 int r; 453 454 do { 455 r = amdgpu_cs_bo_validate(p, bo); 456 } while (r == -ENOMEM && amdgpu_cs_try_evict(p, bo)); 457 if (r) 458 return r; 459 460 if (bo->shadow) 461 r = amdgpu_cs_bo_validate(p, bo->shadow); 462 463 return r; 464 } 465 466 static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, 467 struct list_head *validated) 468 { 469 struct amdgpu_bo_list_entry *lobj; 470 int r; 471 472 list_for_each_entry(lobj, validated, tv.head) { 473 struct amdgpu_bo *bo = lobj->robj; 474 bool binding_userptr = false; 475 struct mm_struct *usermm; 476 477 usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); 478 if (usermm && usermm != current->mm) 479 return -EPERM; 480 481 /* Check if we have user pages and nobody bound the BO already */ 482 if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) && 483 lobj->user_pages) { 484 amdgpu_ttm_placement_from_domain(bo, 485 AMDGPU_GEM_DOMAIN_CPU); 486 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, 487 false); 488 if (r) 489 return r; 490 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, 491 lobj->user_pages); 492 binding_userptr = true; 493 } 494 495 if (p->evictable == lobj) 496 p->evictable = NULL; 497 498 r = amdgpu_cs_validate(p, bo); 499 if (r) 500 return r; 501 502 if (binding_userptr) { 503 kvfree(lobj->user_pages); 504 lobj->user_pages = NULL; 505 } 506 } 507 return 0; 508 } 509 510 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, 511 union drm_amdgpu_cs *cs) 512 { 513 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 514 struct amdgpu_bo_list_entry *e; 515 struct list_head duplicates; 516 unsigned i, tries = 10; 517 int r; 518 519 INIT_LIST_HEAD(&p->validated); 520 521 p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle); 522 if (p->bo_list) { 523 amdgpu_bo_list_get_list(p->bo_list, &p->validated); 524 if (p->bo_list->first_userptr != p->bo_list->num_entries) 525 p->mn = amdgpu_mn_get(p->adev); 526 } 527 528 INIT_LIST_HEAD(&duplicates); 529 amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd); 530 531 if (p->uf_entry.robj) 532 list_add(&p->uf_entry.tv.head, &p->validated); 533 534 while (1) { 535 struct list_head need_pages; 536 unsigned i; 537 538 r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, 539 &duplicates); 540 if (unlikely(r != 0)) { 541 if (r != -ERESTARTSYS) 542 DRM_ERROR("ttm_eu_reserve_buffers failed.\n"); 543 goto error_free_pages; 544 } 545 546 /* Without a BO list we don't have userptr BOs */ 547 if (!p->bo_list) 548 break; 549 550 INIT_LIST_HEAD(&need_pages); 551 for (i = p->bo_list->first_userptr; 552 i < p->bo_list->num_entries; ++i) { 553 struct amdgpu_bo *bo; 554 555 e = &p->bo_list->array[i]; 556 bo = e->robj; 557 558 if (amdgpu_ttm_tt_userptr_invalidated(bo->tbo.ttm, 559 &e->user_invalidated) && e->user_pages) { 560 561 /* We acquired a page array, but somebody 562 * invalidated it. Free it and try again 563 */ 564 release_pages(e->user_pages, 565 bo->tbo.ttm->num_pages); 566 kvfree(e->user_pages); 567 e->user_pages = NULL; 568 } 569 570 if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) && 571 !e->user_pages) { 572 list_del(&e->tv.head); 573 list_add(&e->tv.head, &need_pages); 574 575 amdgpu_bo_unreserve(e->robj); 576 } 577 } 578 579 if (list_empty(&need_pages)) 580 break; 581 582 /* Unreserve everything again. */ 583 ttm_eu_backoff_reservation(&p->ticket, &p->validated); 584 585 /* We tried too many times, just abort */ 586 if (!--tries) { 587 r = -EDEADLK; 588 DRM_ERROR("deadlock in %s\n", __func__); 589 goto error_free_pages; 590 } 591 592 /* Fill the page arrays for all userptrs. */ 593 list_for_each_entry(e, &need_pages, tv.head) { 594 struct ttm_tt *ttm = e->robj->tbo.ttm; 595 596 e->user_pages = kvmalloc_array(ttm->num_pages, 597 sizeof(struct page*), 598 GFP_KERNEL | __GFP_ZERO); 599 if (!e->user_pages) { 600 r = -ENOMEM; 601 DRM_ERROR("calloc failure in %s\n", __func__); 602 goto error_free_pages; 603 } 604 605 r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages); 606 if (r) { 607 DRM_ERROR("amdgpu_ttm_tt_get_user_pages failed.\n"); 608 kvfree(e->user_pages); 609 e->user_pages = NULL; 610 goto error_free_pages; 611 } 612 } 613 614 /* And try again. */ 615 list_splice(&need_pages, &p->validated); 616 } 617 618 amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold, 619 &p->bytes_moved_vis_threshold); 620 p->bytes_moved = 0; 621 p->bytes_moved_vis = 0; 622 p->evictable = list_last_entry(&p->validated, 623 struct amdgpu_bo_list_entry, 624 tv.head); 625 626 r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm, 627 amdgpu_cs_validate, p); 628 if (r) { 629 DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n"); 630 goto error_validate; 631 } 632 633 r = amdgpu_cs_list_validate(p, &duplicates); 634 if (r) { 635 DRM_ERROR("amdgpu_cs_list_validate(duplicates) failed.\n"); 636 goto error_validate; 637 } 638 639 r = amdgpu_cs_list_validate(p, &p->validated); 640 if (r) { 641 DRM_ERROR("amdgpu_cs_list_validate(validated) failed.\n"); 642 goto error_validate; 643 } 644 645 amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved, 646 p->bytes_moved_vis); 647 if (p->bo_list) { 648 struct amdgpu_bo *gds = p->bo_list->gds_obj; 649 struct amdgpu_bo *gws = p->bo_list->gws_obj; 650 struct amdgpu_bo *oa = p->bo_list->oa_obj; 651 struct amdgpu_vm *vm = &fpriv->vm; 652 unsigned i; 653 654 for (i = 0; i < p->bo_list->num_entries; i++) { 655 struct amdgpu_bo *bo = p->bo_list->array[i].robj; 656 657 p->bo_list->array[i].bo_va = amdgpu_vm_bo_find(vm, bo); 658 } 659 660 if (gds) { 661 p->job->gds_base = amdgpu_bo_gpu_offset(gds); 662 p->job->gds_size = amdgpu_bo_size(gds); 663 } 664 if (gws) { 665 p->job->gws_base = amdgpu_bo_gpu_offset(gws); 666 p->job->gws_size = amdgpu_bo_size(gws); 667 } 668 if (oa) { 669 p->job->oa_base = amdgpu_bo_gpu_offset(oa); 670 p->job->oa_size = amdgpu_bo_size(oa); 671 } 672 } 673 674 if (!r && p->uf_entry.robj) { 675 struct amdgpu_bo *uf = p->uf_entry.robj; 676 677 r = amdgpu_ttm_bind(&uf->tbo, &uf->tbo.mem); 678 p->job->uf_addr += amdgpu_bo_gpu_offset(uf); 679 } 680 681 error_validate: 682 if (r) 683 ttm_eu_backoff_reservation(&p->ticket, &p->validated); 684 685 error_free_pages: 686 687 if (p->bo_list) { 688 for (i = p->bo_list->first_userptr; 689 i < p->bo_list->num_entries; ++i) { 690 e = &p->bo_list->array[i]; 691 692 if (!e->user_pages) 693 continue; 694 695 release_pages(e->user_pages, 696 e->robj->tbo.ttm->num_pages); 697 kvfree(e->user_pages); 698 } 699 } 700 701 return r; 702 } 703 704 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) 705 { 706 struct amdgpu_bo_list_entry *e; 707 int r; 708 709 list_for_each_entry(e, &p->validated, tv.head) { 710 struct reservation_object *resv = e->robj->tbo.resv; 711 r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp, 712 amdgpu_bo_explicit_sync(e->robj)); 713 714 if (r) 715 return r; 716 } 717 return 0; 718 } 719 720 /** 721 * cs_parser_fini() - clean parser states 722 * @parser: parser structure holding parsing context. 723 * @error: error number 724 * 725 * If error is set than unvalidate buffer, otherwise just free memory 726 * used by parsing context. 727 **/ 728 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, 729 bool backoff) 730 { 731 unsigned i; 732 733 if (error && backoff) 734 ttm_eu_backoff_reservation(&parser->ticket, 735 &parser->validated); 736 737 for (i = 0; i < parser->num_post_dep_syncobjs; i++) 738 drm_syncobj_put(parser->post_dep_syncobjs[i]); 739 kfree(parser->post_dep_syncobjs); 740 741 dma_fence_put(parser->fence); 742 743 if (parser->ctx) { 744 mutex_unlock(&parser->ctx->lock); 745 amdgpu_ctx_put(parser->ctx); 746 } 747 if (parser->bo_list) 748 amdgpu_bo_list_put(parser->bo_list); 749 750 for (i = 0; i < parser->nchunks; i++) 751 kvfree(parser->chunks[i].kdata); 752 kfree(parser->chunks); 753 if (parser->job) 754 amdgpu_job_free(parser->job); 755 amdgpu_bo_unref(&parser->uf_entry.robj); 756 } 757 758 static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p) 759 { 760 struct amdgpu_device *adev = p->adev; 761 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 762 struct amdgpu_vm *vm = &fpriv->vm; 763 struct amdgpu_bo_va *bo_va; 764 struct amdgpu_bo *bo; 765 int i, r; 766 767 r = amdgpu_vm_update_directories(adev, vm); 768 if (r) 769 return r; 770 771 r = amdgpu_vm_clear_freed(adev, vm, NULL); 772 if (r) 773 return r; 774 775 r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); 776 if (r) 777 return r; 778 779 r = amdgpu_sync_fence(adev, &p->job->sync, 780 fpriv->prt_va->last_pt_update); 781 if (r) 782 return r; 783 784 if (amdgpu_sriov_vf(adev)) { 785 struct dma_fence *f; 786 787 bo_va = fpriv->csa_va; 788 BUG_ON(!bo_va); 789 r = amdgpu_vm_bo_update(adev, bo_va, false); 790 if (r) 791 return r; 792 793 f = bo_va->last_pt_update; 794 r = amdgpu_sync_fence(adev, &p->job->sync, f); 795 if (r) 796 return r; 797 } 798 799 if (p->bo_list) { 800 for (i = 0; i < p->bo_list->num_entries; i++) { 801 struct dma_fence *f; 802 803 /* ignore duplicates */ 804 bo = p->bo_list->array[i].robj; 805 if (!bo) 806 continue; 807 808 bo_va = p->bo_list->array[i].bo_va; 809 if (bo_va == NULL) 810 continue; 811 812 r = amdgpu_vm_bo_update(adev, bo_va, false); 813 if (r) 814 return r; 815 816 f = bo_va->last_pt_update; 817 r = amdgpu_sync_fence(adev, &p->job->sync, f); 818 if (r) 819 return r; 820 } 821 822 } 823 824 r = amdgpu_vm_handle_moved(adev, vm); 825 if (r) 826 return r; 827 828 r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_update); 829 if (r) 830 return r; 831 832 if (amdgpu_vm_debug && p->bo_list) { 833 /* Invalidate all BOs to test for userspace bugs */ 834 for (i = 0; i < p->bo_list->num_entries; i++) { 835 /* ignore duplicates */ 836 bo = p->bo_list->array[i].robj; 837 if (!bo) 838 continue; 839 840 amdgpu_vm_bo_invalidate(adev, bo, false); 841 } 842 } 843 844 return r; 845 } 846 847 static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev, 848 struct amdgpu_cs_parser *p) 849 { 850 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 851 struct amdgpu_vm *vm = &fpriv->vm; 852 struct amdgpu_ring *ring = p->job->ring; 853 int r; 854 855 /* Only for UVD/VCE VM emulation */ 856 if (p->job->ring->funcs->parse_cs) { 857 unsigned i, j; 858 859 for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) { 860 struct drm_amdgpu_cs_chunk_ib *chunk_ib; 861 struct amdgpu_bo_va_mapping *m; 862 struct amdgpu_bo *aobj = NULL; 863 struct amdgpu_cs_chunk *chunk; 864 struct amdgpu_ib *ib; 865 uint64_t offset; 866 uint8_t *kptr; 867 868 chunk = &p->chunks[i]; 869 ib = &p->job->ibs[j]; 870 chunk_ib = chunk->kdata; 871 872 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) 873 continue; 874 875 r = amdgpu_cs_find_mapping(p, chunk_ib->va_start, 876 &aobj, &m); 877 if (r) { 878 DRM_ERROR("IB va_start is invalid\n"); 879 return r; 880 } 881 882 if ((chunk_ib->va_start + chunk_ib->ib_bytes) > 883 (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) { 884 DRM_ERROR("IB va_start+ib_bytes is invalid\n"); 885 return -EINVAL; 886 } 887 888 /* the IB should be reserved at this point */ 889 r = amdgpu_bo_kmap(aobj, (void **)&kptr); 890 if (r) { 891 return r; 892 } 893 894 offset = m->start * AMDGPU_GPU_PAGE_SIZE; 895 kptr += chunk_ib->va_start - offset; 896 897 memcpy(ib->ptr, kptr, chunk_ib->ib_bytes); 898 amdgpu_bo_kunmap(aobj); 899 900 r = amdgpu_ring_parse_cs(ring, p, j); 901 if (r) 902 return r; 903 904 j++; 905 } 906 } 907 908 if (p->job->vm) { 909 p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->root.base.bo); 910 911 r = amdgpu_bo_vm_update_pte(p); 912 if (r) 913 return r; 914 } 915 916 return amdgpu_cs_sync_rings(p); 917 } 918 919 static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, 920 struct amdgpu_cs_parser *parser) 921 { 922 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 923 struct amdgpu_vm *vm = &fpriv->vm; 924 int i, j; 925 int r, ce_preempt = 0, de_preempt = 0; 926 927 for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) { 928 struct amdgpu_cs_chunk *chunk; 929 struct amdgpu_ib *ib; 930 struct drm_amdgpu_cs_chunk_ib *chunk_ib; 931 struct amdgpu_ring *ring; 932 933 chunk = &parser->chunks[i]; 934 ib = &parser->job->ibs[j]; 935 chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata; 936 937 if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) 938 continue; 939 940 if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && amdgpu_sriov_vf(adev)) { 941 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) { 942 if (chunk_ib->flags & AMDGPU_IB_FLAG_CE) 943 ce_preempt++; 944 else 945 de_preempt++; 946 } 947 948 /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */ 949 if (ce_preempt > 1 || de_preempt > 1) 950 return -EINVAL; 951 } 952 953 r = amdgpu_queue_mgr_map(adev, &parser->ctx->queue_mgr, chunk_ib->ip_type, 954 chunk_ib->ip_instance, chunk_ib->ring, &ring); 955 if (r) 956 return r; 957 958 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE) { 959 parser->job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT; 960 if (!parser->ctx->preamble_presented) { 961 parser->job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST; 962 parser->ctx->preamble_presented = true; 963 } 964 } 965 966 if (parser->job->ring && parser->job->ring != ring) 967 return -EINVAL; 968 969 parser->job->ring = ring; 970 971 r = amdgpu_ib_get(adev, vm, 972 ring->funcs->parse_cs ? chunk_ib->ib_bytes : 0, 973 ib); 974 if (r) { 975 DRM_ERROR("Failed to get ib !\n"); 976 return r; 977 } 978 979 ib->gpu_addr = chunk_ib->va_start; 980 ib->length_dw = chunk_ib->ib_bytes / 4; 981 ib->flags = chunk_ib->flags; 982 983 j++; 984 } 985 986 /* UVD & VCE fw doesn't support user fences */ 987 if (parser->job->uf_addr && ( 988 parser->job->ring->funcs->type == AMDGPU_RING_TYPE_UVD || 989 parser->job->ring->funcs->type == AMDGPU_RING_TYPE_VCE)) 990 return -EINVAL; 991 992 return amdgpu_ctx_wait_prev_fence(parser->ctx, parser->job->ring->idx); 993 } 994 995 static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p, 996 struct amdgpu_cs_chunk *chunk) 997 { 998 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 999 unsigned num_deps; 1000 int i, r; 1001 struct drm_amdgpu_cs_chunk_dep *deps; 1002 1003 deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata; 1004 num_deps = chunk->length_dw * 4 / 1005 sizeof(struct drm_amdgpu_cs_chunk_dep); 1006 1007 for (i = 0; i < num_deps; ++i) { 1008 struct amdgpu_ring *ring; 1009 struct amdgpu_ctx *ctx; 1010 struct dma_fence *fence; 1011 1012 ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id); 1013 if (ctx == NULL) 1014 return -EINVAL; 1015 1016 r = amdgpu_queue_mgr_map(p->adev, &ctx->queue_mgr, 1017 deps[i].ip_type, 1018 deps[i].ip_instance, 1019 deps[i].ring, &ring); 1020 if (r) { 1021 amdgpu_ctx_put(ctx); 1022 return r; 1023 } 1024 1025 fence = amdgpu_ctx_get_fence(ctx, ring, 1026 deps[i].handle); 1027 if (IS_ERR(fence)) { 1028 r = PTR_ERR(fence); 1029 amdgpu_ctx_put(ctx); 1030 return r; 1031 } else if (fence) { 1032 r = amdgpu_sync_fence(p->adev, &p->job->sync, 1033 fence); 1034 dma_fence_put(fence); 1035 amdgpu_ctx_put(ctx); 1036 if (r) 1037 return r; 1038 } 1039 } 1040 return 0; 1041 } 1042 1043 static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p, 1044 uint32_t handle) 1045 { 1046 int r; 1047 struct dma_fence *fence; 1048 r = drm_syncobj_find_fence(p->filp, handle, &fence); 1049 if (r) 1050 return r; 1051 1052 r = amdgpu_sync_fence(p->adev, &p->job->sync, fence); 1053 dma_fence_put(fence); 1054 1055 return r; 1056 } 1057 1058 static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser *p, 1059 struct amdgpu_cs_chunk *chunk) 1060 { 1061 unsigned num_deps; 1062 int i, r; 1063 struct drm_amdgpu_cs_chunk_sem *deps; 1064 1065 deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; 1066 num_deps = chunk->length_dw * 4 / 1067 sizeof(struct drm_amdgpu_cs_chunk_sem); 1068 1069 for (i = 0; i < num_deps; ++i) { 1070 r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle); 1071 if (r) 1072 return r; 1073 } 1074 return 0; 1075 } 1076 1077 static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p, 1078 struct amdgpu_cs_chunk *chunk) 1079 { 1080 unsigned num_deps; 1081 int i; 1082 struct drm_amdgpu_cs_chunk_sem *deps; 1083 deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; 1084 num_deps = chunk->length_dw * 4 / 1085 sizeof(struct drm_amdgpu_cs_chunk_sem); 1086 1087 p->post_dep_syncobjs = kmalloc_array(num_deps, 1088 sizeof(struct drm_syncobj *), 1089 GFP_KERNEL); 1090 p->num_post_dep_syncobjs = 0; 1091 1092 if (!p->post_dep_syncobjs) 1093 return -ENOMEM; 1094 1095 for (i = 0; i < num_deps; ++i) { 1096 p->post_dep_syncobjs[i] = drm_syncobj_find(p->filp, deps[i].handle); 1097 if (!p->post_dep_syncobjs[i]) 1098 return -EINVAL; 1099 p->num_post_dep_syncobjs++; 1100 } 1101 return 0; 1102 } 1103 1104 static int amdgpu_cs_dependencies(struct amdgpu_device *adev, 1105 struct amdgpu_cs_parser *p) 1106 { 1107 int i, r; 1108 1109 for (i = 0; i < p->nchunks; ++i) { 1110 struct amdgpu_cs_chunk *chunk; 1111 1112 chunk = &p->chunks[i]; 1113 1114 if (chunk->chunk_id == AMDGPU_CHUNK_ID_DEPENDENCIES) { 1115 r = amdgpu_cs_process_fence_dep(p, chunk); 1116 if (r) 1117 return r; 1118 } else if (chunk->chunk_id == AMDGPU_CHUNK_ID_SYNCOBJ_IN) { 1119 r = amdgpu_cs_process_syncobj_in_dep(p, chunk); 1120 if (r) 1121 return r; 1122 } else if (chunk->chunk_id == AMDGPU_CHUNK_ID_SYNCOBJ_OUT) { 1123 r = amdgpu_cs_process_syncobj_out_dep(p, chunk); 1124 if (r) 1125 return r; 1126 } 1127 } 1128 1129 return 0; 1130 } 1131 1132 static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p) 1133 { 1134 int i; 1135 1136 for (i = 0; i < p->num_post_dep_syncobjs; ++i) 1137 drm_syncobj_replace_fence(p->post_dep_syncobjs[i], p->fence); 1138 } 1139 1140 static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, 1141 union drm_amdgpu_cs *cs) 1142 { 1143 struct amdgpu_ring *ring = p->job->ring; 1144 struct amd_sched_entity *entity = &p->ctx->rings[ring->idx].entity; 1145 struct amdgpu_job *job; 1146 unsigned i; 1147 uint64_t seq; 1148 1149 int r; 1150 1151 amdgpu_mn_lock(p->mn); 1152 if (p->bo_list) { 1153 for (i = p->bo_list->first_userptr; 1154 i < p->bo_list->num_entries; ++i) { 1155 struct amdgpu_bo *bo = p->bo_list->array[i].robj; 1156 1157 if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm)) { 1158 amdgpu_mn_unlock(p->mn); 1159 return -ERESTARTSYS; 1160 } 1161 } 1162 } 1163 1164 job = p->job; 1165 p->job = NULL; 1166 1167 r = amd_sched_job_init(&job->base, &ring->sched, entity, p->filp); 1168 if (r) { 1169 amdgpu_job_free(job); 1170 amdgpu_mn_unlock(p->mn); 1171 return r; 1172 } 1173 1174 job->owner = p->filp; 1175 job->fence_ctx = entity->fence_context; 1176 p->fence = dma_fence_get(&job->base.s_fence->finished); 1177 1178 r = amdgpu_ctx_add_fence(p->ctx, ring, p->fence, &seq); 1179 if (r) { 1180 dma_fence_put(p->fence); 1181 dma_fence_put(&job->base.s_fence->finished); 1182 amdgpu_job_free(job); 1183 amdgpu_mn_unlock(p->mn); 1184 return r; 1185 } 1186 1187 amdgpu_cs_post_dependencies(p); 1188 1189 cs->out.handle = seq; 1190 job->uf_sequence = seq; 1191 1192 amdgpu_job_free_resources(job); 1193 amdgpu_ring_priority_get(job->ring, 1194 amd_sched_get_job_priority(&job->base)); 1195 1196 trace_amdgpu_cs_ioctl(job); 1197 amd_sched_entity_push_job(&job->base); 1198 1199 ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence); 1200 amdgpu_mn_unlock(p->mn); 1201 1202 return 0; 1203 } 1204 1205 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 1206 { 1207 struct amdgpu_device *adev = dev->dev_private; 1208 union drm_amdgpu_cs *cs = data; 1209 struct amdgpu_cs_parser parser = {}; 1210 bool reserved_buffers = false; 1211 int i, r; 1212 1213 if (!adev->accel_working) 1214 return -EBUSY; 1215 1216 parser.adev = adev; 1217 parser.filp = filp; 1218 1219 r = amdgpu_cs_parser_init(&parser, data); 1220 if (r) { 1221 DRM_ERROR("Failed to initialize parser !\n"); 1222 goto out; 1223 } 1224 1225 r = amdgpu_cs_ib_fill(adev, &parser); 1226 if (r) 1227 goto out; 1228 1229 r = amdgpu_cs_parser_bos(&parser, data); 1230 if (r) { 1231 if (r == -ENOMEM) 1232 DRM_ERROR("Not enough memory for command submission!\n"); 1233 else if (r != -ERESTARTSYS) 1234 DRM_ERROR("Failed to process the buffer list %d!\n", r); 1235 goto out; 1236 } 1237 1238 reserved_buffers = true; 1239 1240 r = amdgpu_cs_dependencies(adev, &parser); 1241 if (r) { 1242 DRM_ERROR("Failed in the dependencies handling %d!\n", r); 1243 goto out; 1244 } 1245 1246 for (i = 0; i < parser.job->num_ibs; i++) 1247 trace_amdgpu_cs(&parser, i); 1248 1249 r = amdgpu_cs_ib_vm_chunk(adev, &parser); 1250 if (r) 1251 goto out; 1252 1253 r = amdgpu_cs_submit(&parser, cs); 1254 1255 out: 1256 amdgpu_cs_parser_fini(&parser, r, reserved_buffers); 1257 return r; 1258 } 1259 1260 /** 1261 * amdgpu_cs_wait_ioctl - wait for a command submission to finish 1262 * 1263 * @dev: drm device 1264 * @data: data from userspace 1265 * @filp: file private 1266 * 1267 * Wait for the command submission identified by handle to finish. 1268 */ 1269 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, 1270 struct drm_file *filp) 1271 { 1272 union drm_amdgpu_wait_cs *wait = data; 1273 struct amdgpu_device *adev = dev->dev_private; 1274 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout); 1275 struct amdgpu_ring *ring = NULL; 1276 struct amdgpu_ctx *ctx; 1277 struct dma_fence *fence; 1278 long r; 1279 1280 ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id); 1281 if (ctx == NULL) 1282 return -EINVAL; 1283 1284 r = amdgpu_queue_mgr_map(adev, &ctx->queue_mgr, 1285 wait->in.ip_type, wait->in.ip_instance, 1286 wait->in.ring, &ring); 1287 if (r) { 1288 amdgpu_ctx_put(ctx); 1289 return r; 1290 } 1291 1292 fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle); 1293 if (IS_ERR(fence)) 1294 r = PTR_ERR(fence); 1295 else if (fence) { 1296 r = dma_fence_wait_timeout(fence, true, timeout); 1297 if (r > 0 && fence->error) 1298 r = fence->error; 1299 dma_fence_put(fence); 1300 } else 1301 r = 1; 1302 1303 amdgpu_ctx_put(ctx); 1304 if (r < 0) 1305 return r; 1306 1307 memset(wait, 0, sizeof(*wait)); 1308 wait->out.status = (r == 0); 1309 1310 return 0; 1311 } 1312 1313 /** 1314 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence 1315 * 1316 * @adev: amdgpu device 1317 * @filp: file private 1318 * @user: drm_amdgpu_fence copied from user space 1319 */ 1320 static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev, 1321 struct drm_file *filp, 1322 struct drm_amdgpu_fence *user) 1323 { 1324 struct amdgpu_ring *ring; 1325 struct amdgpu_ctx *ctx; 1326 struct dma_fence *fence; 1327 int r; 1328 1329 ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id); 1330 if (ctx == NULL) 1331 return ERR_PTR(-EINVAL); 1332 1333 r = amdgpu_queue_mgr_map(adev, &ctx->queue_mgr, user->ip_type, 1334 user->ip_instance, user->ring, &ring); 1335 if (r) { 1336 amdgpu_ctx_put(ctx); 1337 return ERR_PTR(r); 1338 } 1339 1340 fence = amdgpu_ctx_get_fence(ctx, ring, user->seq_no); 1341 amdgpu_ctx_put(ctx); 1342 1343 return fence; 1344 } 1345 1346 int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data, 1347 struct drm_file *filp) 1348 { 1349 struct amdgpu_device *adev = dev->dev_private; 1350 union drm_amdgpu_fence_to_handle *info = data; 1351 struct dma_fence *fence; 1352 struct drm_syncobj *syncobj; 1353 struct sync_file *sync_file; 1354 int fd, r; 1355 1356 fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence); 1357 if (IS_ERR(fence)) 1358 return PTR_ERR(fence); 1359 1360 switch (info->in.what) { 1361 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ: 1362 r = drm_syncobj_create(&syncobj, 0, fence); 1363 dma_fence_put(fence); 1364 if (r) 1365 return r; 1366 r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle); 1367 drm_syncobj_put(syncobj); 1368 return r; 1369 1370 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD: 1371 r = drm_syncobj_create(&syncobj, 0, fence); 1372 dma_fence_put(fence); 1373 if (r) 1374 return r; 1375 r = drm_syncobj_get_fd(syncobj, (int*)&info->out.handle); 1376 drm_syncobj_put(syncobj); 1377 return r; 1378 1379 case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD: 1380 fd = get_unused_fd_flags(O_CLOEXEC); 1381 if (fd < 0) { 1382 dma_fence_put(fence); 1383 return fd; 1384 } 1385 1386 sync_file = sync_file_create(fence); 1387 dma_fence_put(fence); 1388 if (!sync_file) { 1389 put_unused_fd(fd); 1390 return -ENOMEM; 1391 } 1392 1393 fd_install(fd, sync_file->file); 1394 info->out.handle = fd; 1395 return 0; 1396 1397 default: 1398 return -EINVAL; 1399 } 1400 } 1401 1402 /** 1403 * amdgpu_cs_wait_all_fence - wait on all fences to signal 1404 * 1405 * @adev: amdgpu device 1406 * @filp: file private 1407 * @wait: wait parameters 1408 * @fences: array of drm_amdgpu_fence 1409 */ 1410 static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev, 1411 struct drm_file *filp, 1412 union drm_amdgpu_wait_fences *wait, 1413 struct drm_amdgpu_fence *fences) 1414 { 1415 uint32_t fence_count = wait->in.fence_count; 1416 unsigned int i; 1417 long r = 1; 1418 1419 for (i = 0; i < fence_count; i++) { 1420 struct dma_fence *fence; 1421 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); 1422 1423 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); 1424 if (IS_ERR(fence)) 1425 return PTR_ERR(fence); 1426 else if (!fence) 1427 continue; 1428 1429 r = dma_fence_wait_timeout(fence, true, timeout); 1430 dma_fence_put(fence); 1431 if (r < 0) 1432 return r; 1433 1434 if (r == 0) 1435 break; 1436 1437 if (fence->error) 1438 return fence->error; 1439 } 1440 1441 memset(wait, 0, sizeof(*wait)); 1442 wait->out.status = (r > 0); 1443 1444 return 0; 1445 } 1446 1447 /** 1448 * amdgpu_cs_wait_any_fence - wait on any fence to signal 1449 * 1450 * @adev: amdgpu device 1451 * @filp: file private 1452 * @wait: wait parameters 1453 * @fences: array of drm_amdgpu_fence 1454 */ 1455 static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev, 1456 struct drm_file *filp, 1457 union drm_amdgpu_wait_fences *wait, 1458 struct drm_amdgpu_fence *fences) 1459 { 1460 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); 1461 uint32_t fence_count = wait->in.fence_count; 1462 uint32_t first = ~0; 1463 struct dma_fence **array; 1464 unsigned int i; 1465 long r; 1466 1467 /* Prepare the fence array */ 1468 array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL); 1469 1470 if (array == NULL) 1471 return -ENOMEM; 1472 1473 for (i = 0; i < fence_count; i++) { 1474 struct dma_fence *fence; 1475 1476 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); 1477 if (IS_ERR(fence)) { 1478 r = PTR_ERR(fence); 1479 goto err_free_fence_array; 1480 } else if (fence) { 1481 array[i] = fence; 1482 } else { /* NULL, the fence has been already signaled */ 1483 r = 1; 1484 first = i; 1485 goto out; 1486 } 1487 } 1488 1489 r = dma_fence_wait_any_timeout(array, fence_count, true, timeout, 1490 &first); 1491 if (r < 0) 1492 goto err_free_fence_array; 1493 1494 out: 1495 memset(wait, 0, sizeof(*wait)); 1496 wait->out.status = (r > 0); 1497 wait->out.first_signaled = first; 1498 1499 if (first < fence_count && array[first]) 1500 r = array[first]->error; 1501 else 1502 r = 0; 1503 1504 err_free_fence_array: 1505 for (i = 0; i < fence_count; i++) 1506 dma_fence_put(array[i]); 1507 kfree(array); 1508 1509 return r; 1510 } 1511 1512 /** 1513 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish 1514 * 1515 * @dev: drm device 1516 * @data: data from userspace 1517 * @filp: file private 1518 */ 1519 int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data, 1520 struct drm_file *filp) 1521 { 1522 struct amdgpu_device *adev = dev->dev_private; 1523 union drm_amdgpu_wait_fences *wait = data; 1524 uint32_t fence_count = wait->in.fence_count; 1525 struct drm_amdgpu_fence *fences_user; 1526 struct drm_amdgpu_fence *fences; 1527 int r; 1528 1529 /* Get the fences from userspace */ 1530 fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence), 1531 GFP_KERNEL); 1532 if (fences == NULL) 1533 return -ENOMEM; 1534 1535 fences_user = u64_to_user_ptr(wait->in.fences); 1536 if (copy_from_user(fences, fences_user, 1537 sizeof(struct drm_amdgpu_fence) * fence_count)) { 1538 r = -EFAULT; 1539 goto err_free_fences; 1540 } 1541 1542 if (wait->in.wait_all) 1543 r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences); 1544 else 1545 r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences); 1546 1547 err_free_fences: 1548 kfree(fences); 1549 1550 return r; 1551 } 1552 1553 /** 1554 * amdgpu_cs_find_bo_va - find bo_va for VM address 1555 * 1556 * @parser: command submission parser context 1557 * @addr: VM address 1558 * @bo: resulting BO of the mapping found 1559 * 1560 * Search the buffer objects in the command submission context for a certain 1561 * virtual memory address. Returns allocation structure when found, NULL 1562 * otherwise. 1563 */ 1564 int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser, 1565 uint64_t addr, struct amdgpu_bo **bo, 1566 struct amdgpu_bo_va_mapping **map) 1567 { 1568 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 1569 struct amdgpu_vm *vm = &fpriv->vm; 1570 struct amdgpu_bo_va_mapping *mapping; 1571 int r; 1572 1573 addr /= AMDGPU_GPU_PAGE_SIZE; 1574 1575 mapping = amdgpu_vm_bo_lookup_mapping(vm, addr); 1576 if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo) 1577 return -EINVAL; 1578 1579 *bo = mapping->bo_va->base.bo; 1580 *map = mapping; 1581 1582 /* Double check that the BO is reserved by this CS */ 1583 if (READ_ONCE((*bo)->tbo.resv->lock.ctx) != &parser->ticket) 1584 return -EINVAL; 1585 1586 if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) { 1587 (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 1588 amdgpu_ttm_placement_from_domain(*bo, (*bo)->allowed_domains); 1589 r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, false, 1590 false); 1591 if (r) 1592 return r; 1593 } 1594 1595 return amdgpu_ttm_bind(&(*bo)->tbo, &(*bo)->tbo.mem); 1596 } 1597