1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2 /************************************************************************** 3 * 4 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 25 * USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 **************************************************************************/ 28 /* 29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 30 */ 31 32 #define pr_fmt(fmt) "[TTM] " fmt 33 34 #include <drm/ttm/ttm_bo_driver.h> 35 #include <drm/ttm/ttm_placement.h> 36 #include <linux/jiffies.h> 37 #include <linux/slab.h> 38 #include <linux/sched.h> 39 #include <linux/mm.h> 40 #include <linux/file.h> 41 #include <linux/module.h> 42 #include <linux/atomic.h> 43 #include <linux/dma-resv.h> 44 45 #include "ttm_module.h" 46 47 /* default destructor */ 48 static void ttm_bo_default_destroy(struct ttm_buffer_object *bo) 49 { 50 kfree(bo); 51 } 52 53 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 54 struct ttm_placement *placement) 55 { 56 struct drm_printer p = drm_debug_printer(TTM_PFX); 57 struct ttm_resource_manager *man; 58 int i, mem_type; 59 60 drm_printf(&p, "No space for %p (%lu pages, %zuK, %zuM)\n", 61 bo, bo->resource->num_pages, bo->base.size >> 10, 62 bo->base.size >> 20); 63 for (i = 0; i < placement->num_placement; i++) { 64 mem_type = placement->placement[i].mem_type; 65 drm_printf(&p, " placement[%d]=0x%08X (%d)\n", 66 i, placement->placement[i].flags, mem_type); 67 man = ttm_manager_type(bo->bdev, mem_type); 68 ttm_resource_manager_debug(man, &p); 69 } 70 } 71 72 static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 73 { 74 struct ttm_device *bdev = bo->bdev; 75 76 list_del_init(&bo->lru); 77 78 if (bdev->funcs->del_from_lru_notify) 79 bdev->funcs->del_from_lru_notify(bo); 80 } 81 82 static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos, 83 struct ttm_buffer_object *bo) 84 { 85 if (!pos->first) 86 pos->first = bo; 87 pos->last = bo; 88 } 89 90 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, 91 struct ttm_resource *mem, 92 struct ttm_lru_bulk_move *bulk) 93 { 94 struct ttm_device *bdev = bo->bdev; 95 struct ttm_resource_manager *man; 96 97 if (!bo->deleted) 98 dma_resv_assert_held(bo->base.resv); 99 100 if (bo->pin_count) { 101 ttm_bo_del_from_lru(bo); 102 return; 103 } 104 105 if (!mem) 106 return; 107 108 man = ttm_manager_type(bdev, mem->mem_type); 109 list_move_tail(&bo->lru, &man->lru[bo->priority]); 110 111 if (bdev->funcs->del_from_lru_notify) 112 bdev->funcs->del_from_lru_notify(bo); 113 114 if (bulk && !bo->pin_count) { 115 switch (bo->resource->mem_type) { 116 case TTM_PL_TT: 117 ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo); 118 break; 119 120 case TTM_PL_VRAM: 121 ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo); 122 break; 123 } 124 } 125 } 126 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); 127 128 void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk) 129 { 130 unsigned i; 131 132 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 133 struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i]; 134 struct ttm_resource_manager *man; 135 136 if (!pos->first) 137 continue; 138 139 dma_resv_assert_held(pos->first->base.resv); 140 dma_resv_assert_held(pos->last->base.resv); 141 142 man = ttm_manager_type(pos->first->bdev, TTM_PL_TT); 143 list_bulk_move_tail(&man->lru[i], &pos->first->lru, 144 &pos->last->lru); 145 } 146 147 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 148 struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i]; 149 struct ttm_resource_manager *man; 150 151 if (!pos->first) 152 continue; 153 154 dma_resv_assert_held(pos->first->base.resv); 155 dma_resv_assert_held(pos->last->base.resv); 156 157 man = ttm_manager_type(pos->first->bdev, TTM_PL_VRAM); 158 list_bulk_move_tail(&man->lru[i], &pos->first->lru, 159 &pos->last->lru); 160 } 161 } 162 EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail); 163 164 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 165 struct ttm_resource *mem, bool evict, 166 struct ttm_operation_ctx *ctx, 167 struct ttm_place *hop) 168 { 169 struct ttm_resource_manager *old_man, *new_man; 170 struct ttm_device *bdev = bo->bdev; 171 int ret; 172 173 old_man = ttm_manager_type(bdev, bo->resource->mem_type); 174 new_man = ttm_manager_type(bdev, mem->mem_type); 175 176 ttm_bo_unmap_virtual(bo); 177 178 /* 179 * Create and bind a ttm if required. 180 */ 181 182 if (new_man->use_tt) { 183 /* Zero init the new TTM structure if the old location should 184 * have used one as well. 185 */ 186 ret = ttm_tt_create(bo, old_man->use_tt); 187 if (ret) 188 goto out_err; 189 190 if (mem->mem_type != TTM_PL_SYSTEM) { 191 ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx); 192 if (ret) 193 goto out_err; 194 } 195 } 196 197 ret = bdev->funcs->move(bo, evict, ctx, mem, hop); 198 if (ret) { 199 if (ret == -EMULTIHOP) 200 return ret; 201 goto out_err; 202 } 203 204 ctx->bytes_moved += bo->base.size; 205 return 0; 206 207 out_err: 208 new_man = ttm_manager_type(bdev, bo->resource->mem_type); 209 if (!new_man->use_tt) 210 ttm_bo_tt_destroy(bo); 211 212 return ret; 213 } 214 215 /* 216 * Call bo::reserved. 217 * Will release GPU memory type usage on destruction. 218 * This is the place to put in driver specific hooks to release 219 * driver private resources. 220 * Will release the bo::reserved lock. 221 */ 222 223 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 224 { 225 if (bo->bdev->funcs->delete_mem_notify) 226 bo->bdev->funcs->delete_mem_notify(bo); 227 228 ttm_bo_tt_destroy(bo); 229 ttm_resource_free(bo, &bo->resource); 230 } 231 232 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) 233 { 234 int r; 235 236 if (bo->base.resv == &bo->base._resv) 237 return 0; 238 239 BUG_ON(!dma_resv_trylock(&bo->base._resv)); 240 241 r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv); 242 dma_resv_unlock(&bo->base._resv); 243 if (r) 244 return r; 245 246 if (bo->type != ttm_bo_type_sg) { 247 /* This works because the BO is about to be destroyed and nobody 248 * reference it any more. The only tricky case is the trylock on 249 * the resv object while holding the lru_lock. 250 */ 251 spin_lock(&bo->bdev->lru_lock); 252 bo->base.resv = &bo->base._resv; 253 spin_unlock(&bo->bdev->lru_lock); 254 } 255 256 return r; 257 } 258 259 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) 260 { 261 struct dma_resv *resv = &bo->base._resv; 262 struct dma_resv_list *fobj; 263 struct dma_fence *fence; 264 int i; 265 266 rcu_read_lock(); 267 fobj = dma_resv_shared_list(resv); 268 fence = dma_resv_excl_fence(resv); 269 if (fence && !fence->ops->signaled) 270 dma_fence_enable_sw_signaling(fence); 271 272 for (i = 0; fobj && i < fobj->shared_count; ++i) { 273 fence = rcu_dereference(fobj->shared[i]); 274 275 if (!fence->ops->signaled) 276 dma_fence_enable_sw_signaling(fence); 277 } 278 rcu_read_unlock(); 279 } 280 281 /** 282 * ttm_bo_cleanup_refs 283 * If bo idle, remove from lru lists, and unref. 284 * If not idle, block if possible. 285 * 286 * Must be called with lru_lock and reservation held, this function 287 * will drop the lru lock and optionally the reservation lock before returning. 288 * 289 * @bo: The buffer object to clean-up 290 * @interruptible: Any sleeps should occur interruptibly. 291 * @no_wait_gpu: Never wait for gpu. Return -EBUSY instead. 292 * @unlock_resv: Unlock the reservation lock as well. 293 */ 294 295 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, 296 bool interruptible, bool no_wait_gpu, 297 bool unlock_resv) 298 { 299 struct dma_resv *resv = &bo->base._resv; 300 int ret; 301 302 if (dma_resv_test_signaled(resv, true)) 303 ret = 0; 304 else 305 ret = -EBUSY; 306 307 if (ret && !no_wait_gpu) { 308 long lret; 309 310 if (unlock_resv) 311 dma_resv_unlock(bo->base.resv); 312 spin_unlock(&bo->bdev->lru_lock); 313 314 lret = dma_resv_wait_timeout(resv, true, interruptible, 315 30 * HZ); 316 317 if (lret < 0) 318 return lret; 319 else if (lret == 0) 320 return -EBUSY; 321 322 spin_lock(&bo->bdev->lru_lock); 323 if (unlock_resv && !dma_resv_trylock(bo->base.resv)) { 324 /* 325 * We raced, and lost, someone else holds the reservation now, 326 * and is probably busy in ttm_bo_cleanup_memtype_use. 327 * 328 * Even if it's not the case, because we finished waiting any 329 * delayed destruction would succeed, so just return success 330 * here. 331 */ 332 spin_unlock(&bo->bdev->lru_lock); 333 return 0; 334 } 335 ret = 0; 336 } 337 338 if (ret || unlikely(list_empty(&bo->ddestroy))) { 339 if (unlock_resv) 340 dma_resv_unlock(bo->base.resv); 341 spin_unlock(&bo->bdev->lru_lock); 342 return ret; 343 } 344 345 ttm_bo_del_from_lru(bo); 346 list_del_init(&bo->ddestroy); 347 spin_unlock(&bo->bdev->lru_lock); 348 ttm_bo_cleanup_memtype_use(bo); 349 350 if (unlock_resv) 351 dma_resv_unlock(bo->base.resv); 352 353 ttm_bo_put(bo); 354 355 return 0; 356 } 357 358 /* 359 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 360 * encountered buffers. 361 */ 362 bool ttm_bo_delayed_delete(struct ttm_device *bdev, bool remove_all) 363 { 364 struct list_head removed; 365 bool empty; 366 367 INIT_LIST_HEAD(&removed); 368 369 spin_lock(&bdev->lru_lock); 370 while (!list_empty(&bdev->ddestroy)) { 371 struct ttm_buffer_object *bo; 372 373 bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, 374 ddestroy); 375 list_move_tail(&bo->ddestroy, &removed); 376 if (!ttm_bo_get_unless_zero(bo)) 377 continue; 378 379 if (remove_all || bo->base.resv != &bo->base._resv) { 380 spin_unlock(&bdev->lru_lock); 381 dma_resv_lock(bo->base.resv, NULL); 382 383 spin_lock(&bdev->lru_lock); 384 ttm_bo_cleanup_refs(bo, false, !remove_all, true); 385 386 } else if (dma_resv_trylock(bo->base.resv)) { 387 ttm_bo_cleanup_refs(bo, false, !remove_all, true); 388 } else { 389 spin_unlock(&bdev->lru_lock); 390 } 391 392 ttm_bo_put(bo); 393 spin_lock(&bdev->lru_lock); 394 } 395 list_splice_tail(&removed, &bdev->ddestroy); 396 empty = list_empty(&bdev->ddestroy); 397 spin_unlock(&bdev->lru_lock); 398 399 return empty; 400 } 401 402 static void ttm_bo_release(struct kref *kref) 403 { 404 struct ttm_buffer_object *bo = 405 container_of(kref, struct ttm_buffer_object, kref); 406 struct ttm_device *bdev = bo->bdev; 407 int ret; 408 409 WARN_ON_ONCE(bo->pin_count); 410 411 if (!bo->deleted) { 412 ret = ttm_bo_individualize_resv(bo); 413 if (ret) { 414 /* Last resort, if we fail to allocate memory for the 415 * fences block for the BO to become idle 416 */ 417 dma_resv_wait_timeout(bo->base.resv, true, false, 418 30 * HZ); 419 } 420 421 if (bo->bdev->funcs->release_notify) 422 bo->bdev->funcs->release_notify(bo); 423 424 drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); 425 ttm_mem_io_free(bdev, bo->resource); 426 } 427 428 if (!dma_resv_test_signaled(bo->base.resv, true) || 429 !dma_resv_trylock(bo->base.resv)) { 430 /* The BO is not idle, resurrect it for delayed destroy */ 431 ttm_bo_flush_all_fences(bo); 432 bo->deleted = true; 433 434 spin_lock(&bo->bdev->lru_lock); 435 436 /* 437 * Make pinned bos immediately available to 438 * shrinkers, now that they are queued for 439 * destruction. 440 * 441 * FIXME: QXL is triggering this. Can be removed when the 442 * driver is fixed. 443 */ 444 if (bo->pin_count) { 445 bo->pin_count = 0; 446 ttm_bo_move_to_lru_tail(bo, bo->resource, NULL); 447 } 448 449 kref_init(&bo->kref); 450 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 451 spin_unlock(&bo->bdev->lru_lock); 452 453 schedule_delayed_work(&bdev->wq, 454 ((HZ / 100) < 1) ? 1 : HZ / 100); 455 return; 456 } 457 458 spin_lock(&bo->bdev->lru_lock); 459 ttm_bo_del_from_lru(bo); 460 list_del(&bo->ddestroy); 461 spin_unlock(&bo->bdev->lru_lock); 462 463 ttm_bo_cleanup_memtype_use(bo); 464 dma_resv_unlock(bo->base.resv); 465 466 atomic_dec(&ttm_glob.bo_count); 467 dma_fence_put(bo->moving); 468 bo->destroy(bo); 469 } 470 471 void ttm_bo_put(struct ttm_buffer_object *bo) 472 { 473 kref_put(&bo->kref, ttm_bo_release); 474 } 475 EXPORT_SYMBOL(ttm_bo_put); 476 477 int ttm_bo_lock_delayed_workqueue(struct ttm_device *bdev) 478 { 479 return cancel_delayed_work_sync(&bdev->wq); 480 } 481 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 482 483 void ttm_bo_unlock_delayed_workqueue(struct ttm_device *bdev, int resched) 484 { 485 if (resched) 486 schedule_delayed_work(&bdev->wq, 487 ((HZ / 100) < 1) ? 1 : HZ / 100); 488 } 489 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 490 491 static int ttm_bo_evict(struct ttm_buffer_object *bo, 492 struct ttm_operation_ctx *ctx) 493 { 494 struct ttm_device *bdev = bo->bdev; 495 struct ttm_resource *evict_mem; 496 struct ttm_placement placement; 497 struct ttm_place hop; 498 int ret = 0; 499 500 memset(&hop, 0, sizeof(hop)); 501 502 dma_resv_assert_held(bo->base.resv); 503 504 placement.num_placement = 0; 505 placement.num_busy_placement = 0; 506 bdev->funcs->evict_flags(bo, &placement); 507 508 if (!placement.num_placement && !placement.num_busy_placement) { 509 ret = ttm_bo_wait(bo, true, false); 510 if (ret) 511 return ret; 512 513 /* 514 * Since we've already synced, this frees backing store 515 * immediately. 516 */ 517 return ttm_bo_pipeline_gutting(bo); 518 } 519 520 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); 521 if (ret) { 522 if (ret != -ERESTARTSYS) { 523 pr_err("Failed to find memory space for buffer 0x%p eviction\n", 524 bo); 525 ttm_bo_mem_space_debug(bo, &placement); 526 } 527 goto out; 528 } 529 530 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); 531 if (unlikely(ret)) { 532 WARN(ret == -EMULTIHOP, "Unexpected multihop in eviction - likely driver bug\n"); 533 if (ret != -ERESTARTSYS) 534 pr_err("Buffer eviction failed\n"); 535 ttm_resource_free(bo, &evict_mem); 536 } 537 out: 538 return ret; 539 } 540 541 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 542 const struct ttm_place *place) 543 { 544 dma_resv_assert_held(bo->base.resv); 545 if (bo->resource->mem_type == TTM_PL_SYSTEM) 546 return true; 547 548 /* Don't evict this BO if it's outside of the 549 * requested placement range 550 */ 551 if (place->fpfn >= (bo->resource->start + bo->resource->num_pages) || 552 (place->lpfn && place->lpfn <= bo->resource->start)) 553 return false; 554 555 return true; 556 } 557 EXPORT_SYMBOL(ttm_bo_eviction_valuable); 558 559 /* 560 * Check the target bo is allowable to be evicted or swapout, including cases: 561 * 562 * a. if share same reservation object with ctx->resv, have assumption 563 * reservation objects should already be locked, so not lock again and 564 * return true directly when either the opreation allow_reserved_eviction 565 * or the target bo already is in delayed free list; 566 * 567 * b. Otherwise, trylock it. 568 */ 569 static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, 570 struct ttm_operation_ctx *ctx, 571 const struct ttm_place *place, 572 bool *locked, bool *busy) 573 { 574 bool ret = false; 575 576 if (bo->base.resv == ctx->resv) { 577 dma_resv_assert_held(bo->base.resv); 578 if (ctx->allow_res_evict) 579 ret = true; 580 *locked = false; 581 if (busy) 582 *busy = false; 583 } else { 584 ret = dma_resv_trylock(bo->base.resv); 585 *locked = ret; 586 if (busy) 587 *busy = !ret; 588 } 589 590 if (ret && place && !bo->bdev->funcs->eviction_valuable(bo, place)) { 591 ret = false; 592 if (*locked) { 593 dma_resv_unlock(bo->base.resv); 594 *locked = false; 595 } 596 } 597 598 return ret; 599 } 600 601 /** 602 * ttm_mem_evict_wait_busy - wait for a busy BO to become available 603 * 604 * @busy_bo: BO which couldn't be locked with trylock 605 * @ctx: operation context 606 * @ticket: acquire ticket 607 * 608 * Try to lock a busy buffer object to avoid failing eviction. 609 */ 610 static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, 611 struct ttm_operation_ctx *ctx, 612 struct ww_acquire_ctx *ticket) 613 { 614 int r; 615 616 if (!busy_bo || !ticket) 617 return -EBUSY; 618 619 if (ctx->interruptible) 620 r = dma_resv_lock_interruptible(busy_bo->base.resv, 621 ticket); 622 else 623 r = dma_resv_lock(busy_bo->base.resv, ticket); 624 625 /* 626 * TODO: It would be better to keep the BO locked until allocation is at 627 * least tried one more time, but that would mean a much larger rework 628 * of TTM. 629 */ 630 if (!r) 631 dma_resv_unlock(busy_bo->base.resv); 632 633 return r == -EDEADLK ? -EBUSY : r; 634 } 635 636 int ttm_mem_evict_first(struct ttm_device *bdev, 637 struct ttm_resource_manager *man, 638 const struct ttm_place *place, 639 struct ttm_operation_ctx *ctx, 640 struct ww_acquire_ctx *ticket) 641 { 642 struct ttm_buffer_object *bo = NULL, *busy_bo = NULL; 643 bool locked = false; 644 unsigned i; 645 int ret; 646 647 spin_lock(&bdev->lru_lock); 648 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 649 list_for_each_entry(bo, &man->lru[i], lru) { 650 bool busy; 651 652 if (!ttm_bo_evict_swapout_allowable(bo, ctx, place, 653 &locked, &busy)) { 654 if (busy && !busy_bo && ticket != 655 dma_resv_locking_ctx(bo->base.resv)) 656 busy_bo = bo; 657 continue; 658 } 659 660 if (!ttm_bo_get_unless_zero(bo)) { 661 if (locked) 662 dma_resv_unlock(bo->base.resv); 663 continue; 664 } 665 break; 666 } 667 668 /* If the inner loop terminated early, we have our candidate */ 669 if (&bo->lru != &man->lru[i]) 670 break; 671 672 bo = NULL; 673 } 674 675 if (!bo) { 676 if (busy_bo && !ttm_bo_get_unless_zero(busy_bo)) 677 busy_bo = NULL; 678 spin_unlock(&bdev->lru_lock); 679 ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket); 680 if (busy_bo) 681 ttm_bo_put(busy_bo); 682 return ret; 683 } 684 685 if (bo->deleted) { 686 ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, 687 ctx->no_wait_gpu, locked); 688 ttm_bo_put(bo); 689 return ret; 690 } 691 692 spin_unlock(&bdev->lru_lock); 693 694 ret = ttm_bo_evict(bo, ctx); 695 if (locked) 696 ttm_bo_unreserve(bo); 697 698 ttm_bo_put(bo); 699 return ret; 700 } 701 702 /* 703 * Add the last move fence to the BO and reserve a new shared slot. We only use 704 * a shared slot to avoid unecessary sync and rely on the subsequent bo move to 705 * either stall or use an exclusive fence respectively set bo->moving. 706 */ 707 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, 708 struct ttm_resource_manager *man, 709 struct ttm_resource *mem, 710 bool no_wait_gpu) 711 { 712 struct dma_fence *fence; 713 int ret; 714 715 spin_lock(&man->move_lock); 716 fence = dma_fence_get(man->move); 717 spin_unlock(&man->move_lock); 718 719 if (!fence) 720 return 0; 721 722 if (no_wait_gpu) { 723 ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY; 724 dma_fence_put(fence); 725 return ret; 726 } 727 728 dma_resv_add_shared_fence(bo->base.resv, fence); 729 730 ret = dma_resv_reserve_shared(bo->base.resv, 1); 731 if (unlikely(ret)) { 732 dma_fence_put(fence); 733 return ret; 734 } 735 736 dma_fence_put(bo->moving); 737 bo->moving = fence; 738 return 0; 739 } 740 741 /* 742 * Repeatedly evict memory from the LRU for @mem_type until we create enough 743 * space, or we've evicted everything and there isn't enough space. 744 */ 745 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 746 const struct ttm_place *place, 747 struct ttm_resource **mem, 748 struct ttm_operation_ctx *ctx) 749 { 750 struct ttm_device *bdev = bo->bdev; 751 struct ttm_resource_manager *man; 752 struct ww_acquire_ctx *ticket; 753 int ret; 754 755 man = ttm_manager_type(bdev, place->mem_type); 756 ticket = dma_resv_locking_ctx(bo->base.resv); 757 do { 758 ret = ttm_resource_alloc(bo, place, mem); 759 if (likely(!ret)) 760 break; 761 if (unlikely(ret != -ENOSPC)) 762 return ret; 763 ret = ttm_mem_evict_first(bdev, man, place, ctx, 764 ticket); 765 if (unlikely(ret != 0)) 766 return ret; 767 } while (1); 768 769 return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu); 770 } 771 772 /* 773 * Creates space for memory region @mem according to its type. 774 * 775 * This function first searches for free space in compatible memory types in 776 * the priority order defined by the driver. If free space isn't found, then 777 * ttm_bo_mem_force_space is attempted in priority order to evict and find 778 * space. 779 */ 780 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 781 struct ttm_placement *placement, 782 struct ttm_resource **mem, 783 struct ttm_operation_ctx *ctx) 784 { 785 struct ttm_device *bdev = bo->bdev; 786 bool type_found = false; 787 int i, ret; 788 789 ret = dma_resv_reserve_shared(bo->base.resv, 1); 790 if (unlikely(ret)) 791 return ret; 792 793 for (i = 0; i < placement->num_placement; ++i) { 794 const struct ttm_place *place = &placement->placement[i]; 795 struct ttm_resource_manager *man; 796 797 man = ttm_manager_type(bdev, place->mem_type); 798 if (!man || !ttm_resource_manager_used(man)) 799 continue; 800 801 type_found = true; 802 ret = ttm_resource_alloc(bo, place, mem); 803 if (ret == -ENOSPC) 804 continue; 805 if (unlikely(ret)) 806 goto error; 807 808 ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu); 809 if (unlikely(ret)) { 810 ttm_resource_free(bo, mem); 811 if (ret == -EBUSY) 812 continue; 813 814 goto error; 815 } 816 return 0; 817 } 818 819 for (i = 0; i < placement->num_busy_placement; ++i) { 820 const struct ttm_place *place = &placement->busy_placement[i]; 821 struct ttm_resource_manager *man; 822 823 man = ttm_manager_type(bdev, place->mem_type); 824 if (!man || !ttm_resource_manager_used(man)) 825 continue; 826 827 type_found = true; 828 ret = ttm_bo_mem_force_space(bo, place, mem, ctx); 829 if (likely(!ret)) 830 return 0; 831 832 if (ret && ret != -EBUSY) 833 goto error; 834 } 835 836 ret = -ENOMEM; 837 if (!type_found) { 838 pr_err(TTM_PFX "No compatible memory type found\n"); 839 ret = -EINVAL; 840 } 841 842 error: 843 if (bo->resource->mem_type == TTM_PL_SYSTEM && !bo->pin_count) 844 ttm_bo_move_to_lru_tail_unlocked(bo); 845 846 return ret; 847 } 848 EXPORT_SYMBOL(ttm_bo_mem_space); 849 850 static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, 851 struct ttm_resource **mem, 852 struct ttm_operation_ctx *ctx, 853 struct ttm_place *hop) 854 { 855 struct ttm_placement hop_placement; 856 struct ttm_resource *hop_mem; 857 int ret; 858 859 hop_placement.num_placement = hop_placement.num_busy_placement = 1; 860 hop_placement.placement = hop_placement.busy_placement = hop; 861 862 /* find space in the bounce domain */ 863 ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx); 864 if (ret) 865 return ret; 866 /* move to the bounce domain */ 867 ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL); 868 if (ret) { 869 ttm_resource_free(bo, &hop_mem); 870 return ret; 871 } 872 return 0; 873 } 874 875 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 876 struct ttm_placement *placement, 877 struct ttm_operation_ctx *ctx) 878 { 879 struct ttm_resource *mem; 880 struct ttm_place hop; 881 int ret; 882 883 dma_resv_assert_held(bo->base.resv); 884 885 /* 886 * Determine where to move the buffer. 887 * 888 * If driver determines move is going to need 889 * an extra step then it will return -EMULTIHOP 890 * and the buffer will be moved to the temporary 891 * stop and the driver will be called to make 892 * the second hop. 893 */ 894 ret = ttm_bo_mem_space(bo, placement, &mem, ctx); 895 if (ret) 896 return ret; 897 bounce: 898 ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop); 899 if (ret == -EMULTIHOP) { 900 ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop); 901 if (ret) 902 goto out; 903 /* try and move to final place now. */ 904 goto bounce; 905 } 906 out: 907 if (ret) 908 ttm_resource_free(bo, &mem); 909 return ret; 910 } 911 912 static bool ttm_bo_places_compat(const struct ttm_place *places, 913 unsigned num_placement, 914 struct ttm_resource *mem, 915 uint32_t *new_flags) 916 { 917 unsigned i; 918 919 for (i = 0; i < num_placement; i++) { 920 const struct ttm_place *heap = &places[i]; 921 922 if ((mem->start < heap->fpfn || 923 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) 924 continue; 925 926 *new_flags = heap->flags; 927 if ((mem->mem_type == heap->mem_type) && 928 (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) || 929 (mem->placement & TTM_PL_FLAG_CONTIGUOUS))) 930 return true; 931 } 932 return false; 933 } 934 935 bool ttm_bo_mem_compat(struct ttm_placement *placement, 936 struct ttm_resource *mem, 937 uint32_t *new_flags) 938 { 939 if (ttm_bo_places_compat(placement->placement, placement->num_placement, 940 mem, new_flags)) 941 return true; 942 943 if ((placement->busy_placement != placement->placement || 944 placement->num_busy_placement > placement->num_placement) && 945 ttm_bo_places_compat(placement->busy_placement, 946 placement->num_busy_placement, 947 mem, new_flags)) 948 return true; 949 950 return false; 951 } 952 EXPORT_SYMBOL(ttm_bo_mem_compat); 953 954 int ttm_bo_validate(struct ttm_buffer_object *bo, 955 struct ttm_placement *placement, 956 struct ttm_operation_ctx *ctx) 957 { 958 int ret; 959 uint32_t new_flags; 960 961 dma_resv_assert_held(bo->base.resv); 962 963 /* 964 * Remove the backing store if no placement is given. 965 */ 966 if (!placement->num_placement && !placement->num_busy_placement) 967 return ttm_bo_pipeline_gutting(bo); 968 969 /* 970 * Check whether we need to move buffer. 971 */ 972 if (!ttm_bo_mem_compat(placement, bo->resource, &new_flags)) { 973 ret = ttm_bo_move_buffer(bo, placement, ctx); 974 if (ret) 975 return ret; 976 } 977 /* 978 * We might need to add a TTM. 979 */ 980 if (bo->resource->mem_type == TTM_PL_SYSTEM) { 981 ret = ttm_tt_create(bo, true); 982 if (ret) 983 return ret; 984 } 985 return 0; 986 } 987 EXPORT_SYMBOL(ttm_bo_validate); 988 989 int ttm_bo_init_reserved(struct ttm_device *bdev, 990 struct ttm_buffer_object *bo, 991 size_t size, 992 enum ttm_bo_type type, 993 struct ttm_placement *placement, 994 uint32_t page_alignment, 995 struct ttm_operation_ctx *ctx, 996 struct sg_table *sg, 997 struct dma_resv *resv, 998 void (*destroy) (struct ttm_buffer_object *)) 999 { 1000 static const struct ttm_place sys_mem = { .mem_type = TTM_PL_SYSTEM }; 1001 bool locked; 1002 int ret; 1003 1004 bo->destroy = destroy ? destroy : ttm_bo_default_destroy; 1005 1006 kref_init(&bo->kref); 1007 INIT_LIST_HEAD(&bo->lru); 1008 INIT_LIST_HEAD(&bo->ddestroy); 1009 bo->bdev = bdev; 1010 bo->type = type; 1011 bo->page_alignment = page_alignment; 1012 bo->moving = NULL; 1013 bo->pin_count = 0; 1014 bo->sg = sg; 1015 if (resv) { 1016 bo->base.resv = resv; 1017 dma_resv_assert_held(bo->base.resv); 1018 } else { 1019 bo->base.resv = &bo->base._resv; 1020 } 1021 atomic_inc(&ttm_glob.bo_count); 1022 1023 ret = ttm_resource_alloc(bo, &sys_mem, &bo->resource); 1024 if (unlikely(ret)) { 1025 ttm_bo_put(bo); 1026 return ret; 1027 } 1028 1029 /* 1030 * For ttm_bo_type_device buffers, allocate 1031 * address space from the device. 1032 */ 1033 if (bo->type == ttm_bo_type_device || 1034 bo->type == ttm_bo_type_sg) 1035 ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node, 1036 bo->resource->num_pages); 1037 1038 /* passed reservation objects should already be locked, 1039 * since otherwise lockdep will be angered in radeon. 1040 */ 1041 if (!resv) { 1042 locked = dma_resv_trylock(bo->base.resv); 1043 WARN_ON(!locked); 1044 } 1045 1046 if (likely(!ret)) 1047 ret = ttm_bo_validate(bo, placement, ctx); 1048 1049 if (unlikely(ret)) { 1050 if (!resv) 1051 ttm_bo_unreserve(bo); 1052 1053 ttm_bo_put(bo); 1054 return ret; 1055 } 1056 1057 ttm_bo_move_to_lru_tail_unlocked(bo); 1058 1059 return ret; 1060 } 1061 EXPORT_SYMBOL(ttm_bo_init_reserved); 1062 1063 int ttm_bo_init(struct ttm_device *bdev, 1064 struct ttm_buffer_object *bo, 1065 size_t size, 1066 enum ttm_bo_type type, 1067 struct ttm_placement *placement, 1068 uint32_t page_alignment, 1069 bool interruptible, 1070 struct sg_table *sg, 1071 struct dma_resv *resv, 1072 void (*destroy) (struct ttm_buffer_object *)) 1073 { 1074 struct ttm_operation_ctx ctx = { interruptible, false }; 1075 int ret; 1076 1077 ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, 1078 page_alignment, &ctx, sg, resv, destroy); 1079 if (ret) 1080 return ret; 1081 1082 if (!resv) 1083 ttm_bo_unreserve(bo); 1084 1085 return 0; 1086 } 1087 EXPORT_SYMBOL(ttm_bo_init); 1088 1089 /* 1090 * buffer object vm functions. 1091 */ 1092 1093 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1094 { 1095 struct ttm_device *bdev = bo->bdev; 1096 1097 drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); 1098 ttm_mem_io_free(bdev, bo->resource); 1099 } 1100 EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1101 1102 int ttm_bo_wait(struct ttm_buffer_object *bo, 1103 bool interruptible, bool no_wait) 1104 { 1105 long timeout = 15 * HZ; 1106 1107 if (no_wait) { 1108 if (dma_resv_test_signaled(bo->base.resv, true)) 1109 return 0; 1110 else 1111 return -EBUSY; 1112 } 1113 1114 timeout = dma_resv_wait_timeout(bo->base.resv, true, interruptible, 1115 timeout); 1116 if (timeout < 0) 1117 return timeout; 1118 1119 if (timeout == 0) 1120 return -EBUSY; 1121 1122 dma_resv_add_excl_fence(bo->base.resv, NULL); 1123 return 0; 1124 } 1125 EXPORT_SYMBOL(ttm_bo_wait); 1126 1127 int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, 1128 gfp_t gfp_flags) 1129 { 1130 struct ttm_place place; 1131 bool locked; 1132 int ret; 1133 1134 /* 1135 * While the bo may already reside in SYSTEM placement, set 1136 * SYSTEM as new placement to cover also the move further below. 1137 * The driver may use the fact that we're moving from SYSTEM 1138 * as an indication that we're about to swap out. 1139 */ 1140 memset(&place, 0, sizeof(place)); 1141 place.mem_type = TTM_PL_SYSTEM; 1142 if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL)) 1143 return -EBUSY; 1144 1145 if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) || 1146 bo->ttm->page_flags & TTM_PAGE_FLAG_SG || 1147 bo->ttm->page_flags & TTM_PAGE_FLAG_SWAPPED || 1148 !ttm_bo_get_unless_zero(bo)) { 1149 if (locked) 1150 dma_resv_unlock(bo->base.resv); 1151 return -EBUSY; 1152 } 1153 1154 if (bo->deleted) { 1155 ttm_bo_cleanup_refs(bo, false, false, locked); 1156 ttm_bo_put(bo); 1157 return 0; 1158 } 1159 1160 ttm_bo_del_from_lru(bo); 1161 /* TODO: Cleanup the locking */ 1162 spin_unlock(&bo->bdev->lru_lock); 1163 1164 /* 1165 * Move to system cached 1166 */ 1167 if (bo->resource->mem_type != TTM_PL_SYSTEM) { 1168 struct ttm_operation_ctx ctx = { false, false }; 1169 struct ttm_resource *evict_mem; 1170 struct ttm_place hop; 1171 1172 memset(&hop, 0, sizeof(hop)); 1173 ret = ttm_resource_alloc(bo, &place, &evict_mem); 1174 if (unlikely(ret)) 1175 goto out; 1176 1177 ret = ttm_bo_handle_move_mem(bo, evict_mem, true, &ctx, &hop); 1178 if (unlikely(ret != 0)) { 1179 WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n"); 1180 goto out; 1181 } 1182 } 1183 1184 /* 1185 * Make sure BO is idle. 1186 */ 1187 ret = ttm_bo_wait(bo, false, false); 1188 if (unlikely(ret != 0)) 1189 goto out; 1190 1191 ttm_bo_unmap_virtual(bo); 1192 1193 /* 1194 * Swap out. Buffer will be swapped in again as soon as 1195 * anyone tries to access a ttm page. 1196 */ 1197 if (bo->bdev->funcs->swap_notify) 1198 bo->bdev->funcs->swap_notify(bo); 1199 1200 if (ttm_tt_is_populated(bo->ttm)) 1201 ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags); 1202 out: 1203 1204 /* 1205 * Unreserve without putting on LRU to avoid swapping out an 1206 * already swapped buffer. 1207 */ 1208 if (locked) 1209 dma_resv_unlock(bo->base.resv); 1210 ttm_bo_put(bo); 1211 return ret; 1212 } 1213 1214 void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) 1215 { 1216 if (bo->ttm == NULL) 1217 return; 1218 1219 ttm_tt_destroy(bo->bdev, bo->ttm); 1220 bo->ttm = NULL; 1221 } 1222