1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #define pr_fmt(fmt) "[TTM] " fmt 32 33 #include <drm/ttm/ttm_module.h> 34 #include <drm/ttm/ttm_bo_driver.h> 35 #include <drm/ttm/ttm_placement.h> 36 #include <linux/jiffies.h> 37 #include <linux/slab.h> 38 #include <linux/sched.h> 39 #include <linux/mm.h> 40 #include <linux/file.h> 41 #include <linux/module.h> 42 #include <linux/atomic.h> 43 #include <linux/reservation.h> 44 45 #define TTM_ASSERT_LOCKED(param) 46 #define TTM_DEBUG(fmt, arg...) 47 #define TTM_BO_HASH_ORDER 13 48 49 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); 50 static void ttm_bo_global_kobj_release(struct kobject *kobj); 51 52 static struct attribute ttm_bo_count = { 53 .name = "bo_count", 54 .mode = S_IRUGO 55 }; 56 57 static inline int ttm_mem_type_from_place(const struct ttm_place *place, 58 uint32_t *mem_type) 59 { 60 int pos; 61 62 pos = ffs(place->flags & TTM_PL_MASK_MEM); 63 if (unlikely(!pos)) 64 return -EINVAL; 65 66 *mem_type = pos - 1; 67 return 0; 68 } 69 70 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 71 { 72 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 73 struct drm_printer p = drm_debug_printer(TTM_PFX); 74 75 pr_err(" has_type: %d\n", man->has_type); 76 pr_err(" use_type: %d\n", man->use_type); 77 pr_err(" flags: 0x%08X\n", man->flags); 78 pr_err(" gpu_offset: 0x%08llX\n", man->gpu_offset); 79 pr_err(" size: %llu\n", man->size); 80 pr_err(" available_caching: 0x%08X\n", man->available_caching); 81 pr_err(" default_caching: 0x%08X\n", man->default_caching); 82 if (mem_type != TTM_PL_SYSTEM) 83 (*man->func->debug)(man, &p); 84 } 85 86 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 87 struct ttm_placement *placement) 88 { 89 int i, ret, mem_type; 90 91 pr_err("No space for %p (%lu pages, %luK, %luM)\n", 92 bo, bo->mem.num_pages, bo->mem.size >> 10, 93 bo->mem.size >> 20); 94 for (i = 0; i < placement->num_placement; i++) { 95 ret = ttm_mem_type_from_place(&placement->placement[i], 96 &mem_type); 97 if (ret) 98 return; 99 pr_err(" placement[%d]=0x%08X (%d)\n", 100 i, placement->placement[i].flags, mem_type); 101 ttm_mem_type_debug(bo->bdev, mem_type); 102 } 103 } 104 105 static ssize_t ttm_bo_global_show(struct kobject *kobj, 106 struct attribute *attr, 107 char *buffer) 108 { 109 struct ttm_bo_global *glob = 110 container_of(kobj, struct ttm_bo_global, kobj); 111 112 return snprintf(buffer, PAGE_SIZE, "%d\n", 113 atomic_read(&glob->bo_count)); 114 } 115 116 static struct attribute *ttm_bo_global_attrs[] = { 117 &ttm_bo_count, 118 NULL 119 }; 120 121 static const struct sysfs_ops ttm_bo_global_ops = { 122 .show = &ttm_bo_global_show 123 }; 124 125 static struct kobj_type ttm_bo_glob_kobj_type = { 126 .release = &ttm_bo_global_kobj_release, 127 .sysfs_ops = &ttm_bo_global_ops, 128 .default_attrs = ttm_bo_global_attrs 129 }; 130 131 132 static inline uint32_t ttm_bo_type_flags(unsigned type) 133 { 134 return 1 << (type); 135 } 136 137 static void ttm_bo_release_list(struct kref *list_kref) 138 { 139 struct ttm_buffer_object *bo = 140 container_of(list_kref, struct ttm_buffer_object, list_kref); 141 struct ttm_bo_device *bdev = bo->bdev; 142 size_t acc_size = bo->acc_size; 143 144 BUG_ON(kref_read(&bo->list_kref)); 145 BUG_ON(kref_read(&bo->kref)); 146 BUG_ON(atomic_read(&bo->cpu_writers)); 147 BUG_ON(bo->mem.mm_node != NULL); 148 BUG_ON(!list_empty(&bo->lru)); 149 BUG_ON(!list_empty(&bo->ddestroy)); 150 ttm_tt_destroy(bo->ttm); 151 atomic_dec(&bo->glob->bo_count); 152 dma_fence_put(bo->moving); 153 reservation_object_fini(&bo->ttm_resv); 154 mutex_destroy(&bo->wu_mutex); 155 if (bo->destroy) 156 bo->destroy(bo); 157 else { 158 kfree(bo); 159 } 160 ttm_mem_global_free(bdev->glob->mem_glob, acc_size); 161 } 162 163 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 164 { 165 struct ttm_bo_device *bdev = bo->bdev; 166 struct ttm_mem_type_manager *man; 167 168 lockdep_assert_held(&bo->resv->lock.base); 169 170 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 171 172 BUG_ON(!list_empty(&bo->lru)); 173 174 man = &bdev->man[bo->mem.mem_type]; 175 list_add_tail(&bo->lru, &man->lru[bo->priority]); 176 kref_get(&bo->list_kref); 177 178 if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { 179 list_add_tail(&bo->swap, 180 &bo->glob->swap_lru[bo->priority]); 181 kref_get(&bo->list_kref); 182 } 183 } 184 } 185 EXPORT_SYMBOL(ttm_bo_add_to_lru); 186 187 static void ttm_bo_ref_bug(struct kref *list_kref) 188 { 189 BUG(); 190 } 191 192 void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 193 { 194 if (!list_empty(&bo->swap)) { 195 list_del_init(&bo->swap); 196 kref_put(&bo->list_kref, ttm_bo_ref_bug); 197 } 198 if (!list_empty(&bo->lru)) { 199 list_del_init(&bo->lru); 200 kref_put(&bo->list_kref, ttm_bo_ref_bug); 201 } 202 203 /* 204 * TODO: Add a driver hook to delete from 205 * driver-specific LRU's here. 206 */ 207 } 208 209 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) 210 { 211 spin_lock(&bo->glob->lru_lock); 212 ttm_bo_del_from_lru(bo); 213 spin_unlock(&bo->glob->lru_lock); 214 } 215 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); 216 217 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) 218 { 219 lockdep_assert_held(&bo->resv->lock.base); 220 221 ttm_bo_del_from_lru(bo); 222 ttm_bo_add_to_lru(bo); 223 } 224 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); 225 226 /* 227 * Call bo->mutex locked. 228 */ 229 static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 230 { 231 struct ttm_bo_device *bdev = bo->bdev; 232 struct ttm_bo_global *glob = bo->glob; 233 int ret = 0; 234 uint32_t page_flags = 0; 235 236 TTM_ASSERT_LOCKED(&bo->mutex); 237 bo->ttm = NULL; 238 239 if (bdev->need_dma32) 240 page_flags |= TTM_PAGE_FLAG_DMA32; 241 242 switch (bo->type) { 243 case ttm_bo_type_device: 244 if (zero_alloc) 245 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 246 case ttm_bo_type_kernel: 247 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 248 page_flags, glob->dummy_read_page); 249 if (unlikely(bo->ttm == NULL)) 250 ret = -ENOMEM; 251 break; 252 case ttm_bo_type_sg: 253 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 254 page_flags | TTM_PAGE_FLAG_SG, 255 glob->dummy_read_page); 256 if (unlikely(bo->ttm == NULL)) { 257 ret = -ENOMEM; 258 break; 259 } 260 bo->ttm->sg = bo->sg; 261 break; 262 default: 263 pr_err("Illegal buffer object type\n"); 264 ret = -EINVAL; 265 break; 266 } 267 268 return ret; 269 } 270 271 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 272 struct ttm_mem_reg *mem, 273 bool evict, bool interruptible, 274 bool no_wait_gpu) 275 { 276 struct ttm_bo_device *bdev = bo->bdev; 277 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 278 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 279 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 280 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 281 int ret = 0; 282 283 if (old_is_pci || new_is_pci || 284 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 285 ret = ttm_mem_io_lock(old_man, true); 286 if (unlikely(ret != 0)) 287 goto out_err; 288 ttm_bo_unmap_virtual_locked(bo); 289 ttm_mem_io_unlock(old_man); 290 } 291 292 /* 293 * Create and bind a ttm if required. 294 */ 295 296 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 297 if (bo->ttm == NULL) { 298 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 299 ret = ttm_bo_add_ttm(bo, zero); 300 if (ret) 301 goto out_err; 302 } 303 304 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 305 if (ret) 306 goto out_err; 307 308 if (mem->mem_type != TTM_PL_SYSTEM) { 309 ret = ttm_tt_bind(bo->ttm, mem); 310 if (ret) 311 goto out_err; 312 } 313 314 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 315 if (bdev->driver->move_notify) 316 bdev->driver->move_notify(bo, evict, mem); 317 bo->mem = *mem; 318 mem->mm_node = NULL; 319 goto moved; 320 } 321 } 322 323 if (bdev->driver->move_notify) 324 bdev->driver->move_notify(bo, evict, mem); 325 326 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 327 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 328 ret = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, mem); 329 else if (bdev->driver->move) 330 ret = bdev->driver->move(bo, evict, interruptible, 331 no_wait_gpu, mem); 332 else 333 ret = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, mem); 334 335 if (ret) { 336 if (bdev->driver->move_notify) { 337 struct ttm_mem_reg tmp_mem = *mem; 338 *mem = bo->mem; 339 bo->mem = tmp_mem; 340 bdev->driver->move_notify(bo, false, mem); 341 bo->mem = *mem; 342 *mem = tmp_mem; 343 } 344 345 goto out_err; 346 } 347 348 moved: 349 if (bo->evicted) { 350 if (bdev->driver->invalidate_caches) { 351 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 352 if (ret) 353 pr_err("Can not flush read caches\n"); 354 } 355 bo->evicted = false; 356 } 357 358 if (bo->mem.mm_node) { 359 bo->offset = (bo->mem.start << PAGE_SHIFT) + 360 bdev->man[bo->mem.mem_type].gpu_offset; 361 bo->cur_placement = bo->mem.placement; 362 } else 363 bo->offset = 0; 364 365 return 0; 366 367 out_err: 368 new_man = &bdev->man[bo->mem.mem_type]; 369 if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) { 370 ttm_tt_destroy(bo->ttm); 371 bo->ttm = NULL; 372 } 373 374 return ret; 375 } 376 377 /** 378 * Call bo::reserved. 379 * Will release GPU memory type usage on destruction. 380 * This is the place to put in driver specific hooks to release 381 * driver private resources. 382 * Will release the bo::reserved lock. 383 */ 384 385 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 386 { 387 if (bo->bdev->driver->move_notify) 388 bo->bdev->driver->move_notify(bo, false, NULL); 389 390 ttm_tt_destroy(bo->ttm); 391 bo->ttm = NULL; 392 ttm_bo_mem_put(bo, &bo->mem); 393 394 ww_mutex_unlock (&bo->resv->lock); 395 } 396 397 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) 398 { 399 int r; 400 401 if (bo->resv == &bo->ttm_resv) 402 return 0; 403 404 BUG_ON(!reservation_object_trylock(&bo->ttm_resv)); 405 406 r = reservation_object_copy_fences(&bo->ttm_resv, bo->resv); 407 if (r) 408 reservation_object_unlock(&bo->ttm_resv); 409 410 return r; 411 } 412 413 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) 414 { 415 struct reservation_object_list *fobj; 416 struct dma_fence *fence; 417 int i; 418 419 fobj = reservation_object_get_list(&bo->ttm_resv); 420 fence = reservation_object_get_excl(&bo->ttm_resv); 421 if (fence && !fence->ops->signaled) 422 dma_fence_enable_sw_signaling(fence); 423 424 for (i = 0; fobj && i < fobj->shared_count; ++i) { 425 fence = rcu_dereference_protected(fobj->shared[i], 426 reservation_object_held(bo->resv)); 427 428 if (!fence->ops->signaled) 429 dma_fence_enable_sw_signaling(fence); 430 } 431 } 432 433 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 434 { 435 struct ttm_bo_device *bdev = bo->bdev; 436 struct ttm_bo_global *glob = bo->glob; 437 int ret; 438 439 ret = ttm_bo_individualize_resv(bo); 440 if (ret) { 441 /* Last resort, if we fail to allocate memory for the 442 * fences block for the BO to become idle 443 */ 444 reservation_object_wait_timeout_rcu(bo->resv, true, false, 445 30 * HZ); 446 spin_lock(&glob->lru_lock); 447 goto error; 448 } 449 450 spin_lock(&glob->lru_lock); 451 ret = __ttm_bo_reserve(bo, false, true, NULL); 452 if (!ret) { 453 if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) { 454 ttm_bo_del_from_lru(bo); 455 spin_unlock(&glob->lru_lock); 456 if (bo->resv != &bo->ttm_resv) 457 reservation_object_unlock(&bo->ttm_resv); 458 459 ttm_bo_cleanup_memtype_use(bo); 460 return; 461 } 462 463 ttm_bo_flush_all_fences(bo); 464 465 /* 466 * Make NO_EVICT bos immediately available to 467 * shrinkers, now that they are queued for 468 * destruction. 469 */ 470 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { 471 bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; 472 ttm_bo_add_to_lru(bo); 473 } 474 475 __ttm_bo_unreserve(bo); 476 } 477 if (bo->resv != &bo->ttm_resv) 478 reservation_object_unlock(&bo->ttm_resv); 479 480 error: 481 kref_get(&bo->list_kref); 482 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 483 spin_unlock(&glob->lru_lock); 484 485 schedule_delayed_work(&bdev->wq, 486 ((HZ / 100) < 1) ? 1 : HZ / 100); 487 } 488 489 /** 490 * function ttm_bo_cleanup_refs_and_unlock 491 * If bo idle, remove from delayed- and lru lists, and unref. 492 * If not idle, do nothing. 493 * 494 * Must be called with lru_lock and reservation held, this function 495 * will drop both before returning. 496 * 497 * @interruptible Any sleeps should occur interruptibly. 498 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 499 */ 500 501 static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, 502 bool interruptible, 503 bool no_wait_gpu) 504 { 505 struct ttm_bo_global *glob = bo->glob; 506 struct reservation_object *resv; 507 int ret; 508 509 if (unlikely(list_empty(&bo->ddestroy))) 510 resv = bo->resv; 511 else 512 resv = &bo->ttm_resv; 513 514 if (reservation_object_test_signaled_rcu(resv, true)) 515 ret = 0; 516 else 517 ret = -EBUSY; 518 519 if (ret && !no_wait_gpu) { 520 long lret; 521 ww_mutex_unlock(&bo->resv->lock); 522 spin_unlock(&glob->lru_lock); 523 524 lret = reservation_object_wait_timeout_rcu(resv, true, 525 interruptible, 526 30 * HZ); 527 528 if (lret < 0) 529 return lret; 530 else if (lret == 0) 531 return -EBUSY; 532 533 spin_lock(&glob->lru_lock); 534 ret = __ttm_bo_reserve(bo, false, true, NULL); 535 536 /* 537 * We raced, and lost, someone else holds the reservation now, 538 * and is probably busy in ttm_bo_cleanup_memtype_use. 539 * 540 * Even if it's not the case, because we finished waiting any 541 * delayed destruction would succeed, so just return success 542 * here. 543 */ 544 if (ret) { 545 spin_unlock(&glob->lru_lock); 546 return 0; 547 } 548 } 549 550 if (ret || unlikely(list_empty(&bo->ddestroy))) { 551 __ttm_bo_unreserve(bo); 552 spin_unlock(&glob->lru_lock); 553 return ret; 554 } 555 556 ttm_bo_del_from_lru(bo); 557 list_del_init(&bo->ddestroy); 558 kref_put(&bo->list_kref, ttm_bo_ref_bug); 559 560 spin_unlock(&glob->lru_lock); 561 ttm_bo_cleanup_memtype_use(bo); 562 563 return 0; 564 } 565 566 /** 567 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 568 * encountered buffers. 569 */ 570 571 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 572 { 573 struct ttm_bo_global *glob = bdev->glob; 574 struct ttm_buffer_object *entry = NULL; 575 int ret = 0; 576 577 spin_lock(&glob->lru_lock); 578 if (list_empty(&bdev->ddestroy)) 579 goto out_unlock; 580 581 entry = list_first_entry(&bdev->ddestroy, 582 struct ttm_buffer_object, ddestroy); 583 kref_get(&entry->list_kref); 584 585 for (;;) { 586 struct ttm_buffer_object *nentry = NULL; 587 588 if (entry->ddestroy.next != &bdev->ddestroy) { 589 nentry = list_first_entry(&entry->ddestroy, 590 struct ttm_buffer_object, ddestroy); 591 kref_get(&nentry->list_kref); 592 } 593 594 ret = __ttm_bo_reserve(entry, false, true, NULL); 595 if (remove_all && ret) { 596 spin_unlock(&glob->lru_lock); 597 ret = __ttm_bo_reserve(entry, false, false, NULL); 598 spin_lock(&glob->lru_lock); 599 } 600 601 if (!ret) 602 ret = ttm_bo_cleanup_refs_and_unlock(entry, false, 603 !remove_all); 604 else 605 spin_unlock(&glob->lru_lock); 606 607 kref_put(&entry->list_kref, ttm_bo_release_list); 608 entry = nentry; 609 610 if (ret || !entry) 611 goto out; 612 613 spin_lock(&glob->lru_lock); 614 if (list_empty(&entry->ddestroy)) 615 break; 616 } 617 618 out_unlock: 619 spin_unlock(&glob->lru_lock); 620 out: 621 if (entry) 622 kref_put(&entry->list_kref, ttm_bo_release_list); 623 return ret; 624 } 625 626 static void ttm_bo_delayed_workqueue(struct work_struct *work) 627 { 628 struct ttm_bo_device *bdev = 629 container_of(work, struct ttm_bo_device, wq.work); 630 631 if (ttm_bo_delayed_delete(bdev, false)) { 632 schedule_delayed_work(&bdev->wq, 633 ((HZ / 100) < 1) ? 1 : HZ / 100); 634 } 635 } 636 637 static void ttm_bo_release(struct kref *kref) 638 { 639 struct ttm_buffer_object *bo = 640 container_of(kref, struct ttm_buffer_object, kref); 641 struct ttm_bo_device *bdev = bo->bdev; 642 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 643 644 drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); 645 ttm_mem_io_lock(man, false); 646 ttm_mem_io_free_vm(bo); 647 ttm_mem_io_unlock(man); 648 ttm_bo_cleanup_refs_or_queue(bo); 649 kref_put(&bo->list_kref, ttm_bo_release_list); 650 } 651 652 void ttm_bo_unref(struct ttm_buffer_object **p_bo) 653 { 654 struct ttm_buffer_object *bo = *p_bo; 655 656 *p_bo = NULL; 657 kref_put(&bo->kref, ttm_bo_release); 658 } 659 EXPORT_SYMBOL(ttm_bo_unref); 660 661 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 662 { 663 return cancel_delayed_work_sync(&bdev->wq); 664 } 665 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 666 667 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 668 { 669 if (resched) 670 schedule_delayed_work(&bdev->wq, 671 ((HZ / 100) < 1) ? 1 : HZ / 100); 672 } 673 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 674 675 static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 676 bool no_wait_gpu) 677 { 678 struct ttm_bo_device *bdev = bo->bdev; 679 struct ttm_mem_reg evict_mem; 680 struct ttm_placement placement; 681 int ret = 0; 682 683 lockdep_assert_held(&bo->resv->lock.base); 684 685 evict_mem = bo->mem; 686 evict_mem.mm_node = NULL; 687 evict_mem.bus.io_reserved_vm = false; 688 evict_mem.bus.io_reserved_count = 0; 689 690 placement.num_placement = 0; 691 placement.num_busy_placement = 0; 692 bdev->driver->evict_flags(bo, &placement); 693 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 694 no_wait_gpu); 695 if (ret) { 696 if (ret != -ERESTARTSYS) { 697 pr_err("Failed to find memory space for buffer 0x%p eviction\n", 698 bo); 699 ttm_bo_mem_space_debug(bo, &placement); 700 } 701 goto out; 702 } 703 704 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 705 no_wait_gpu); 706 if (unlikely(ret)) { 707 if (ret != -ERESTARTSYS) 708 pr_err("Buffer eviction failed\n"); 709 ttm_bo_mem_put(bo, &evict_mem); 710 goto out; 711 } 712 bo->evicted = true; 713 out: 714 return ret; 715 } 716 717 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 718 const struct ttm_place *place) 719 { 720 /* Don't evict this BO if it's outside of the 721 * requested placement range 722 */ 723 if (place->fpfn >= (bo->mem.start + bo->mem.size) || 724 (place->lpfn && place->lpfn <= bo->mem.start)) 725 return false; 726 727 return true; 728 } 729 EXPORT_SYMBOL(ttm_bo_eviction_valuable); 730 731 static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 732 uint32_t mem_type, 733 const struct ttm_place *place, 734 bool interruptible, 735 bool no_wait_gpu) 736 { 737 struct ttm_bo_global *glob = bdev->glob; 738 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 739 struct ttm_buffer_object *bo; 740 int ret = -EBUSY; 741 unsigned i; 742 743 spin_lock(&glob->lru_lock); 744 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 745 list_for_each_entry(bo, &man->lru[i], lru) { 746 ret = __ttm_bo_reserve(bo, false, true, NULL); 747 if (ret) 748 continue; 749 750 if (place && !bdev->driver->eviction_valuable(bo, 751 place)) { 752 __ttm_bo_unreserve(bo); 753 ret = -EBUSY; 754 continue; 755 } 756 757 break; 758 } 759 760 if (!ret) 761 break; 762 } 763 764 if (ret) { 765 spin_unlock(&glob->lru_lock); 766 return ret; 767 } 768 769 kref_get(&bo->list_kref); 770 771 if (!list_empty(&bo->ddestroy)) { 772 ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, 773 no_wait_gpu); 774 kref_put(&bo->list_kref, ttm_bo_release_list); 775 return ret; 776 } 777 778 ttm_bo_del_from_lru(bo); 779 spin_unlock(&glob->lru_lock); 780 781 BUG_ON(ret != 0); 782 783 ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); 784 ttm_bo_unreserve(bo); 785 786 kref_put(&bo->list_kref, ttm_bo_release_list); 787 return ret; 788 } 789 790 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 791 { 792 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 793 794 if (mem->mm_node) 795 (*man->func->put_node)(man, mem); 796 } 797 EXPORT_SYMBOL(ttm_bo_mem_put); 798 799 /** 800 * Add the last move fence to the BO and reserve a new shared slot. 801 */ 802 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, 803 struct ttm_mem_type_manager *man, 804 struct ttm_mem_reg *mem) 805 { 806 struct dma_fence *fence; 807 int ret; 808 809 spin_lock(&man->move_lock); 810 fence = dma_fence_get(man->move); 811 spin_unlock(&man->move_lock); 812 813 if (fence) { 814 reservation_object_add_shared_fence(bo->resv, fence); 815 816 ret = reservation_object_reserve_shared(bo->resv); 817 if (unlikely(ret)) 818 return ret; 819 820 dma_fence_put(bo->moving); 821 bo->moving = fence; 822 } 823 824 return 0; 825 } 826 827 /** 828 * Repeatedly evict memory from the LRU for @mem_type until we create enough 829 * space, or we've evicted everything and there isn't enough space. 830 */ 831 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 832 uint32_t mem_type, 833 const struct ttm_place *place, 834 struct ttm_mem_reg *mem, 835 bool interruptible, 836 bool no_wait_gpu) 837 { 838 struct ttm_bo_device *bdev = bo->bdev; 839 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 840 int ret; 841 842 do { 843 ret = (*man->func->get_node)(man, bo, place, mem); 844 if (unlikely(ret != 0)) 845 return ret; 846 if (mem->mm_node) 847 break; 848 ret = ttm_mem_evict_first(bdev, mem_type, place, 849 interruptible, no_wait_gpu); 850 if (unlikely(ret != 0)) 851 return ret; 852 } while (1); 853 mem->mem_type = mem_type; 854 return ttm_bo_add_move_fence(bo, man, mem); 855 } 856 857 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 858 uint32_t cur_placement, 859 uint32_t proposed_placement) 860 { 861 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 862 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 863 864 /** 865 * Keep current caching if possible. 866 */ 867 868 if ((cur_placement & caching) != 0) 869 result |= (cur_placement & caching); 870 else if ((man->default_caching & caching) != 0) 871 result |= man->default_caching; 872 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 873 result |= TTM_PL_FLAG_CACHED; 874 else if ((TTM_PL_FLAG_WC & caching) != 0) 875 result |= TTM_PL_FLAG_WC; 876 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 877 result |= TTM_PL_FLAG_UNCACHED; 878 879 return result; 880 } 881 882 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 883 uint32_t mem_type, 884 const struct ttm_place *place, 885 uint32_t *masked_placement) 886 { 887 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 888 889 if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0) 890 return false; 891 892 if ((place->flags & man->available_caching) == 0) 893 return false; 894 895 cur_flags |= (place->flags & man->available_caching); 896 897 *masked_placement = cur_flags; 898 return true; 899 } 900 901 /** 902 * Creates space for memory region @mem according to its type. 903 * 904 * This function first searches for free space in compatible memory types in 905 * the priority order defined by the driver. If free space isn't found, then 906 * ttm_bo_mem_force_space is attempted in priority order to evict and find 907 * space. 908 */ 909 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 910 struct ttm_placement *placement, 911 struct ttm_mem_reg *mem, 912 bool interruptible, 913 bool no_wait_gpu) 914 { 915 struct ttm_bo_device *bdev = bo->bdev; 916 struct ttm_mem_type_manager *man; 917 uint32_t mem_type = TTM_PL_SYSTEM; 918 uint32_t cur_flags = 0; 919 bool type_found = false; 920 bool type_ok = false; 921 bool has_erestartsys = false; 922 int i, ret; 923 924 ret = reservation_object_reserve_shared(bo->resv); 925 if (unlikely(ret)) 926 return ret; 927 928 mem->mm_node = NULL; 929 for (i = 0; i < placement->num_placement; ++i) { 930 const struct ttm_place *place = &placement->placement[i]; 931 932 ret = ttm_mem_type_from_place(place, &mem_type); 933 if (ret) 934 return ret; 935 man = &bdev->man[mem_type]; 936 if (!man->has_type || !man->use_type) 937 continue; 938 939 type_ok = ttm_bo_mt_compatible(man, mem_type, place, 940 &cur_flags); 941 942 if (!type_ok) 943 continue; 944 945 type_found = true; 946 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 947 cur_flags); 948 /* 949 * Use the access and other non-mapping-related flag bits from 950 * the memory placement flags to the current flags 951 */ 952 ttm_flag_masked(&cur_flags, place->flags, 953 ~TTM_PL_MASK_MEMTYPE); 954 955 if (mem_type == TTM_PL_SYSTEM) 956 break; 957 958 ret = (*man->func->get_node)(man, bo, place, mem); 959 if (unlikely(ret)) 960 return ret; 961 962 if (mem->mm_node) { 963 ret = ttm_bo_add_move_fence(bo, man, mem); 964 if (unlikely(ret)) { 965 (*man->func->put_node)(man, mem); 966 return ret; 967 } 968 break; 969 } 970 } 971 972 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 973 mem->mem_type = mem_type; 974 mem->placement = cur_flags; 975 return 0; 976 } 977 978 for (i = 0; i < placement->num_busy_placement; ++i) { 979 const struct ttm_place *place = &placement->busy_placement[i]; 980 981 ret = ttm_mem_type_from_place(place, &mem_type); 982 if (ret) 983 return ret; 984 man = &bdev->man[mem_type]; 985 if (!man->has_type || !man->use_type) 986 continue; 987 if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags)) 988 continue; 989 990 type_found = true; 991 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 992 cur_flags); 993 /* 994 * Use the access and other non-mapping-related flag bits from 995 * the memory placement flags to the current flags 996 */ 997 ttm_flag_masked(&cur_flags, place->flags, 998 ~TTM_PL_MASK_MEMTYPE); 999 1000 if (mem_type == TTM_PL_SYSTEM) { 1001 mem->mem_type = mem_type; 1002 mem->placement = cur_flags; 1003 mem->mm_node = NULL; 1004 return 0; 1005 } 1006 1007 ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, 1008 interruptible, no_wait_gpu); 1009 if (ret == 0 && mem->mm_node) { 1010 mem->placement = cur_flags; 1011 return 0; 1012 } 1013 if (ret == -ERESTARTSYS) 1014 has_erestartsys = true; 1015 } 1016 1017 if (!type_found) { 1018 pr_err(TTM_PFX "No compatible memory type found\n"); 1019 return -EINVAL; 1020 } 1021 1022 return (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; 1023 } 1024 EXPORT_SYMBOL(ttm_bo_mem_space); 1025 1026 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 1027 struct ttm_placement *placement, 1028 bool interruptible, 1029 bool no_wait_gpu) 1030 { 1031 int ret = 0; 1032 struct ttm_mem_reg mem; 1033 1034 lockdep_assert_held(&bo->resv->lock.base); 1035 1036 mem.num_pages = bo->num_pages; 1037 mem.size = mem.num_pages << PAGE_SHIFT; 1038 mem.page_alignment = bo->mem.page_alignment; 1039 mem.bus.io_reserved_vm = false; 1040 mem.bus.io_reserved_count = 0; 1041 /* 1042 * Determine where to move the buffer. 1043 */ 1044 ret = ttm_bo_mem_space(bo, placement, &mem, 1045 interruptible, no_wait_gpu); 1046 if (ret) 1047 goto out_unlock; 1048 ret = ttm_bo_handle_move_mem(bo, &mem, false, 1049 interruptible, no_wait_gpu); 1050 out_unlock: 1051 if (ret && mem.mm_node) 1052 ttm_bo_mem_put(bo, &mem); 1053 return ret; 1054 } 1055 1056 static bool ttm_bo_places_compat(const struct ttm_place *places, 1057 unsigned num_placement, 1058 struct ttm_mem_reg *mem, 1059 uint32_t *new_flags) 1060 { 1061 unsigned i; 1062 1063 for (i = 0; i < num_placement; i++) { 1064 const struct ttm_place *heap = &places[i]; 1065 1066 if (mem->mm_node && (mem->start < heap->fpfn || 1067 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) 1068 continue; 1069 1070 *new_flags = heap->flags; 1071 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1072 (*new_flags & mem->placement & TTM_PL_MASK_MEM) && 1073 (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) || 1074 (mem->placement & TTM_PL_FLAG_CONTIGUOUS))) 1075 return true; 1076 } 1077 return false; 1078 } 1079 1080 bool ttm_bo_mem_compat(struct ttm_placement *placement, 1081 struct ttm_mem_reg *mem, 1082 uint32_t *new_flags) 1083 { 1084 if (ttm_bo_places_compat(placement->placement, placement->num_placement, 1085 mem, new_flags)) 1086 return true; 1087 1088 if ((placement->busy_placement != placement->placement || 1089 placement->num_busy_placement > placement->num_placement) && 1090 ttm_bo_places_compat(placement->busy_placement, 1091 placement->num_busy_placement, 1092 mem, new_flags)) 1093 return true; 1094 1095 return false; 1096 } 1097 EXPORT_SYMBOL(ttm_bo_mem_compat); 1098 1099 int ttm_bo_validate(struct ttm_buffer_object *bo, 1100 struct ttm_placement *placement, 1101 bool interruptible, 1102 bool no_wait_gpu) 1103 { 1104 int ret; 1105 uint32_t new_flags; 1106 1107 lockdep_assert_held(&bo->resv->lock.base); 1108 /* 1109 * Check whether we need to move buffer. 1110 */ 1111 if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { 1112 ret = ttm_bo_move_buffer(bo, placement, interruptible, 1113 no_wait_gpu); 1114 if (ret) 1115 return ret; 1116 } else { 1117 /* 1118 * Use the access and other non-mapping-related flag bits from 1119 * the compatible memory placement flags to the active flags 1120 */ 1121 ttm_flag_masked(&bo->mem.placement, new_flags, 1122 ~TTM_PL_MASK_MEMTYPE); 1123 } 1124 /* 1125 * We might need to add a TTM. 1126 */ 1127 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1128 ret = ttm_bo_add_ttm(bo, true); 1129 if (ret) 1130 return ret; 1131 } 1132 return 0; 1133 } 1134 EXPORT_SYMBOL(ttm_bo_validate); 1135 1136 int ttm_bo_init_reserved(struct ttm_bo_device *bdev, 1137 struct ttm_buffer_object *bo, 1138 unsigned long size, 1139 enum ttm_bo_type type, 1140 struct ttm_placement *placement, 1141 uint32_t page_alignment, 1142 bool interruptible, 1143 struct file *persistent_swap_storage, 1144 size_t acc_size, 1145 struct sg_table *sg, 1146 struct reservation_object *resv, 1147 void (*destroy) (struct ttm_buffer_object *)) 1148 { 1149 int ret = 0; 1150 unsigned long num_pages; 1151 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1152 bool locked; 1153 1154 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1155 if (ret) { 1156 pr_err("Out of kernel memory\n"); 1157 if (destroy) 1158 (*destroy)(bo); 1159 else 1160 kfree(bo); 1161 return -ENOMEM; 1162 } 1163 1164 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1165 if (num_pages == 0) { 1166 pr_err("Illegal buffer object size\n"); 1167 if (destroy) 1168 (*destroy)(bo); 1169 else 1170 kfree(bo); 1171 ttm_mem_global_free(mem_glob, acc_size); 1172 return -EINVAL; 1173 } 1174 bo->destroy = destroy; 1175 1176 kref_init(&bo->kref); 1177 kref_init(&bo->list_kref); 1178 atomic_set(&bo->cpu_writers, 0); 1179 INIT_LIST_HEAD(&bo->lru); 1180 INIT_LIST_HEAD(&bo->ddestroy); 1181 INIT_LIST_HEAD(&bo->swap); 1182 INIT_LIST_HEAD(&bo->io_reserve_lru); 1183 mutex_init(&bo->wu_mutex); 1184 bo->bdev = bdev; 1185 bo->glob = bdev->glob; 1186 bo->type = type; 1187 bo->num_pages = num_pages; 1188 bo->mem.size = num_pages << PAGE_SHIFT; 1189 bo->mem.mem_type = TTM_PL_SYSTEM; 1190 bo->mem.num_pages = bo->num_pages; 1191 bo->mem.mm_node = NULL; 1192 bo->mem.page_alignment = page_alignment; 1193 bo->mem.bus.io_reserved_vm = false; 1194 bo->mem.bus.io_reserved_count = 0; 1195 bo->moving = NULL; 1196 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1197 bo->persistent_swap_storage = persistent_swap_storage; 1198 bo->acc_size = acc_size; 1199 bo->sg = sg; 1200 if (resv) { 1201 bo->resv = resv; 1202 lockdep_assert_held(&bo->resv->lock.base); 1203 } else { 1204 bo->resv = &bo->ttm_resv; 1205 } 1206 reservation_object_init(&bo->ttm_resv); 1207 atomic_inc(&bo->glob->bo_count); 1208 drm_vma_node_reset(&bo->vma_node); 1209 bo->priority = 0; 1210 1211 /* 1212 * For ttm_bo_type_device buffers, allocate 1213 * address space from the device. 1214 */ 1215 if (bo->type == ttm_bo_type_device || 1216 bo->type == ttm_bo_type_sg) 1217 ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, 1218 bo->mem.num_pages); 1219 1220 /* passed reservation objects should already be locked, 1221 * since otherwise lockdep will be angered in radeon. 1222 */ 1223 if (!resv) { 1224 locked = ww_mutex_trylock(&bo->resv->lock); 1225 WARN_ON(!locked); 1226 } 1227 1228 if (likely(!ret)) 1229 ret = ttm_bo_validate(bo, placement, interruptible, false); 1230 1231 if (unlikely(ret)) { 1232 if (!resv) 1233 ttm_bo_unreserve(bo); 1234 1235 ttm_bo_unref(&bo); 1236 return ret; 1237 } 1238 1239 if (resv && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 1240 spin_lock(&bo->glob->lru_lock); 1241 ttm_bo_add_to_lru(bo); 1242 spin_unlock(&bo->glob->lru_lock); 1243 } 1244 1245 return ret; 1246 } 1247 EXPORT_SYMBOL(ttm_bo_init_reserved); 1248 1249 int ttm_bo_init(struct ttm_bo_device *bdev, 1250 struct ttm_buffer_object *bo, 1251 unsigned long size, 1252 enum ttm_bo_type type, 1253 struct ttm_placement *placement, 1254 uint32_t page_alignment, 1255 bool interruptible, 1256 struct file *persistent_swap_storage, 1257 size_t acc_size, 1258 struct sg_table *sg, 1259 struct reservation_object *resv, 1260 void (*destroy) (struct ttm_buffer_object *)) 1261 { 1262 int ret; 1263 1264 ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, 1265 page_alignment, interruptible, 1266 persistent_swap_storage, acc_size, 1267 sg, resv, destroy); 1268 if (ret) 1269 return ret; 1270 1271 if (!resv) 1272 ttm_bo_unreserve(bo); 1273 1274 return 0; 1275 } 1276 EXPORT_SYMBOL(ttm_bo_init); 1277 1278 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1279 unsigned long bo_size, 1280 unsigned struct_size) 1281 { 1282 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1283 size_t size = 0; 1284 1285 size += ttm_round_pot(struct_size); 1286 size += ttm_round_pot(npages * sizeof(void *)); 1287 size += ttm_round_pot(sizeof(struct ttm_tt)); 1288 return size; 1289 } 1290 EXPORT_SYMBOL(ttm_bo_acc_size); 1291 1292 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1293 unsigned long bo_size, 1294 unsigned struct_size) 1295 { 1296 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1297 size_t size = 0; 1298 1299 size += ttm_round_pot(struct_size); 1300 size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t))); 1301 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1302 return size; 1303 } 1304 EXPORT_SYMBOL(ttm_bo_dma_acc_size); 1305 1306 int ttm_bo_create(struct ttm_bo_device *bdev, 1307 unsigned long size, 1308 enum ttm_bo_type type, 1309 struct ttm_placement *placement, 1310 uint32_t page_alignment, 1311 bool interruptible, 1312 struct file *persistent_swap_storage, 1313 struct ttm_buffer_object **p_bo) 1314 { 1315 struct ttm_buffer_object *bo; 1316 size_t acc_size; 1317 int ret; 1318 1319 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1320 if (unlikely(bo == NULL)) 1321 return -ENOMEM; 1322 1323 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1324 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1325 interruptible, persistent_swap_storage, acc_size, 1326 NULL, NULL, NULL); 1327 if (likely(ret == 0)) 1328 *p_bo = bo; 1329 1330 return ret; 1331 } 1332 EXPORT_SYMBOL(ttm_bo_create); 1333 1334 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1335 unsigned mem_type) 1336 { 1337 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1338 struct ttm_bo_global *glob = bdev->glob; 1339 struct dma_fence *fence; 1340 int ret; 1341 unsigned i; 1342 1343 /* 1344 * Can't use standard list traversal since we're unlocking. 1345 */ 1346 1347 spin_lock(&glob->lru_lock); 1348 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 1349 while (!list_empty(&man->lru[i])) { 1350 spin_unlock(&glob->lru_lock); 1351 ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false); 1352 if (ret) 1353 return ret; 1354 spin_lock(&glob->lru_lock); 1355 } 1356 } 1357 spin_unlock(&glob->lru_lock); 1358 1359 spin_lock(&man->move_lock); 1360 fence = dma_fence_get(man->move); 1361 spin_unlock(&man->move_lock); 1362 1363 if (fence) { 1364 ret = dma_fence_wait(fence, false); 1365 dma_fence_put(fence); 1366 if (ret) 1367 return ret; 1368 } 1369 1370 return 0; 1371 } 1372 1373 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1374 { 1375 struct ttm_mem_type_manager *man; 1376 int ret = -EINVAL; 1377 1378 if (mem_type >= TTM_NUM_MEM_TYPES) { 1379 pr_err("Illegal memory type %d\n", mem_type); 1380 return ret; 1381 } 1382 man = &bdev->man[mem_type]; 1383 1384 if (!man->has_type) { 1385 pr_err("Trying to take down uninitialized memory manager type %u\n", 1386 mem_type); 1387 return ret; 1388 } 1389 1390 man->use_type = false; 1391 man->has_type = false; 1392 1393 ret = 0; 1394 if (mem_type > 0) { 1395 ret = ttm_bo_force_list_clean(bdev, mem_type); 1396 if (ret) { 1397 pr_err("Cleanup eviction failed\n"); 1398 return ret; 1399 } 1400 1401 ret = (*man->func->takedown)(man); 1402 } 1403 1404 dma_fence_put(man->move); 1405 man->move = NULL; 1406 1407 return ret; 1408 } 1409 EXPORT_SYMBOL(ttm_bo_clean_mm); 1410 1411 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1412 { 1413 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1414 1415 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1416 pr_err("Illegal memory manager memory type %u\n", mem_type); 1417 return -EINVAL; 1418 } 1419 1420 if (!man->has_type) { 1421 pr_err("Memory type %u has not been initialized\n", mem_type); 1422 return 0; 1423 } 1424 1425 return ttm_bo_force_list_clean(bdev, mem_type); 1426 } 1427 EXPORT_SYMBOL(ttm_bo_evict_mm); 1428 1429 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1430 unsigned long p_size) 1431 { 1432 int ret; 1433 struct ttm_mem_type_manager *man; 1434 unsigned i; 1435 1436 BUG_ON(type >= TTM_NUM_MEM_TYPES); 1437 man = &bdev->man[type]; 1438 BUG_ON(man->has_type); 1439 man->io_reserve_fastpath = true; 1440 man->use_io_reserve_lru = false; 1441 mutex_init(&man->io_reserve_mutex); 1442 spin_lock_init(&man->move_lock); 1443 INIT_LIST_HEAD(&man->io_reserve_lru); 1444 1445 ret = bdev->driver->init_mem_type(bdev, type, man); 1446 if (ret) 1447 return ret; 1448 man->bdev = bdev; 1449 1450 if (type != TTM_PL_SYSTEM) { 1451 ret = (*man->func->init)(man, p_size); 1452 if (ret) 1453 return ret; 1454 } 1455 man->has_type = true; 1456 man->use_type = true; 1457 man->size = p_size; 1458 1459 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1460 INIT_LIST_HEAD(&man->lru[i]); 1461 man->move = NULL; 1462 1463 return 0; 1464 } 1465 EXPORT_SYMBOL(ttm_bo_init_mm); 1466 1467 static void ttm_bo_global_kobj_release(struct kobject *kobj) 1468 { 1469 struct ttm_bo_global *glob = 1470 container_of(kobj, struct ttm_bo_global, kobj); 1471 1472 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1473 __free_page(glob->dummy_read_page); 1474 kfree(glob); 1475 } 1476 1477 void ttm_bo_global_release(struct drm_global_reference *ref) 1478 { 1479 struct ttm_bo_global *glob = ref->object; 1480 1481 kobject_del(&glob->kobj); 1482 kobject_put(&glob->kobj); 1483 } 1484 EXPORT_SYMBOL(ttm_bo_global_release); 1485 1486 int ttm_bo_global_init(struct drm_global_reference *ref) 1487 { 1488 struct ttm_bo_global_ref *bo_ref = 1489 container_of(ref, struct ttm_bo_global_ref, ref); 1490 struct ttm_bo_global *glob = ref->object; 1491 int ret; 1492 unsigned i; 1493 1494 mutex_init(&glob->device_list_mutex); 1495 spin_lock_init(&glob->lru_lock); 1496 glob->mem_glob = bo_ref->mem_glob; 1497 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); 1498 1499 if (unlikely(glob->dummy_read_page == NULL)) { 1500 ret = -ENOMEM; 1501 goto out_no_drp; 1502 } 1503 1504 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1505 INIT_LIST_HEAD(&glob->swap_lru[i]); 1506 INIT_LIST_HEAD(&glob->device_list); 1507 1508 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1509 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1510 if (unlikely(ret != 0)) { 1511 pr_err("Could not register buffer object swapout\n"); 1512 goto out_no_shrink; 1513 } 1514 1515 atomic_set(&glob->bo_count, 0); 1516 1517 ret = kobject_init_and_add( 1518 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); 1519 if (unlikely(ret != 0)) 1520 kobject_put(&glob->kobj); 1521 return ret; 1522 out_no_shrink: 1523 __free_page(glob->dummy_read_page); 1524 out_no_drp: 1525 kfree(glob); 1526 return ret; 1527 } 1528 EXPORT_SYMBOL(ttm_bo_global_init); 1529 1530 1531 int ttm_bo_device_release(struct ttm_bo_device *bdev) 1532 { 1533 int ret = 0; 1534 unsigned i = TTM_NUM_MEM_TYPES; 1535 struct ttm_mem_type_manager *man; 1536 struct ttm_bo_global *glob = bdev->glob; 1537 1538 while (i--) { 1539 man = &bdev->man[i]; 1540 if (man->has_type) { 1541 man->use_type = false; 1542 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1543 ret = -EBUSY; 1544 pr_err("DRM memory manager type %d is not clean\n", 1545 i); 1546 } 1547 man->has_type = false; 1548 } 1549 } 1550 1551 mutex_lock(&glob->device_list_mutex); 1552 list_del(&bdev->device_list); 1553 mutex_unlock(&glob->device_list_mutex); 1554 1555 cancel_delayed_work_sync(&bdev->wq); 1556 1557 while (ttm_bo_delayed_delete(bdev, true)) 1558 ; 1559 1560 spin_lock(&glob->lru_lock); 1561 if (list_empty(&bdev->ddestroy)) 1562 TTM_DEBUG("Delayed destroy list was clean\n"); 1563 1564 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 1565 if (list_empty(&bdev->man[0].lru[0])) 1566 TTM_DEBUG("Swap list %d was clean\n", i); 1567 spin_unlock(&glob->lru_lock); 1568 1569 drm_vma_offset_manager_destroy(&bdev->vma_manager); 1570 1571 return ret; 1572 } 1573 EXPORT_SYMBOL(ttm_bo_device_release); 1574 1575 int ttm_bo_device_init(struct ttm_bo_device *bdev, 1576 struct ttm_bo_global *glob, 1577 struct ttm_bo_driver *driver, 1578 struct address_space *mapping, 1579 uint64_t file_page_offset, 1580 bool need_dma32) 1581 { 1582 int ret = -EINVAL; 1583 1584 bdev->driver = driver; 1585 1586 memset(bdev->man, 0, sizeof(bdev->man)); 1587 1588 /* 1589 * Initialize the system memory buffer type. 1590 * Other types need to be driver / IOCTL initialized. 1591 */ 1592 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1593 if (unlikely(ret != 0)) 1594 goto out_no_sys; 1595 1596 drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, 1597 0x10000000); 1598 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); 1599 INIT_LIST_HEAD(&bdev->ddestroy); 1600 bdev->dev_mapping = mapping; 1601 bdev->glob = glob; 1602 bdev->need_dma32 = need_dma32; 1603 mutex_lock(&glob->device_list_mutex); 1604 list_add_tail(&bdev->device_list, &glob->device_list); 1605 mutex_unlock(&glob->device_list_mutex); 1606 1607 return 0; 1608 out_no_sys: 1609 return ret; 1610 } 1611 EXPORT_SYMBOL(ttm_bo_device_init); 1612 1613 /* 1614 * buffer object vm functions. 1615 */ 1616 1617 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1618 { 1619 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1620 1621 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1622 if (mem->mem_type == TTM_PL_SYSTEM) 1623 return false; 1624 1625 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1626 return false; 1627 1628 if (mem->placement & TTM_PL_FLAG_CACHED) 1629 return false; 1630 } 1631 return true; 1632 } 1633 1634 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1635 { 1636 struct ttm_bo_device *bdev = bo->bdev; 1637 1638 drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); 1639 ttm_mem_io_free_vm(bo); 1640 } 1641 1642 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1643 { 1644 struct ttm_bo_device *bdev = bo->bdev; 1645 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1646 1647 ttm_mem_io_lock(man, false); 1648 ttm_bo_unmap_virtual_locked(bo); 1649 ttm_mem_io_unlock(man); 1650 } 1651 1652 1653 EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1654 1655 int ttm_bo_wait(struct ttm_buffer_object *bo, 1656 bool interruptible, bool no_wait) 1657 { 1658 long timeout = 15 * HZ; 1659 1660 if (no_wait) { 1661 if (reservation_object_test_signaled_rcu(bo->resv, true)) 1662 return 0; 1663 else 1664 return -EBUSY; 1665 } 1666 1667 timeout = reservation_object_wait_timeout_rcu(bo->resv, true, 1668 interruptible, timeout); 1669 if (timeout < 0) 1670 return timeout; 1671 1672 if (timeout == 0) 1673 return -EBUSY; 1674 1675 reservation_object_add_excl_fence(bo->resv, NULL); 1676 return 0; 1677 } 1678 EXPORT_SYMBOL(ttm_bo_wait); 1679 1680 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1681 { 1682 int ret = 0; 1683 1684 /* 1685 * Using ttm_bo_reserve makes sure the lru lists are updated. 1686 */ 1687 1688 ret = ttm_bo_reserve(bo, true, no_wait, NULL); 1689 if (unlikely(ret != 0)) 1690 return ret; 1691 ret = ttm_bo_wait(bo, true, no_wait); 1692 if (likely(ret == 0)) 1693 atomic_inc(&bo->cpu_writers); 1694 ttm_bo_unreserve(bo); 1695 return ret; 1696 } 1697 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); 1698 1699 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1700 { 1701 atomic_dec(&bo->cpu_writers); 1702 } 1703 EXPORT_SYMBOL(ttm_bo_synccpu_write_release); 1704 1705 /** 1706 * A buffer object shrink method that tries to swap out the first 1707 * buffer object on the bo_global::swap_lru list. 1708 */ 1709 1710 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1711 { 1712 struct ttm_bo_global *glob = 1713 container_of(shrink, struct ttm_bo_global, shrink); 1714 struct ttm_buffer_object *bo; 1715 int ret = -EBUSY; 1716 unsigned i; 1717 1718 spin_lock(&glob->lru_lock); 1719 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 1720 list_for_each_entry(bo, &glob->swap_lru[i], swap) { 1721 ret = __ttm_bo_reserve(bo, false, true, NULL); 1722 if (!ret) 1723 break; 1724 } 1725 if (!ret) 1726 break; 1727 } 1728 1729 if (ret) { 1730 spin_unlock(&glob->lru_lock); 1731 return ret; 1732 } 1733 1734 kref_get(&bo->list_kref); 1735 1736 if (!list_empty(&bo->ddestroy)) { 1737 ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); 1738 kref_put(&bo->list_kref, ttm_bo_release_list); 1739 return ret; 1740 } 1741 1742 ttm_bo_del_from_lru(bo); 1743 spin_unlock(&glob->lru_lock); 1744 1745 /** 1746 * Move to system cached 1747 */ 1748 1749 if (bo->mem.mem_type != TTM_PL_SYSTEM || 1750 bo->ttm->caching_state != tt_cached) { 1751 struct ttm_mem_reg evict_mem; 1752 1753 evict_mem = bo->mem; 1754 evict_mem.mm_node = NULL; 1755 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1756 evict_mem.mem_type = TTM_PL_SYSTEM; 1757 1758 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1759 false, false); 1760 if (unlikely(ret != 0)) 1761 goto out; 1762 } 1763 1764 /** 1765 * Make sure BO is idle. 1766 */ 1767 1768 ret = ttm_bo_wait(bo, false, false); 1769 if (unlikely(ret != 0)) 1770 goto out; 1771 1772 ttm_bo_unmap_virtual(bo); 1773 1774 /** 1775 * Swap out. Buffer will be swapped in again as soon as 1776 * anyone tries to access a ttm page. 1777 */ 1778 1779 if (bo->bdev->driver->swap_notify) 1780 bo->bdev->driver->swap_notify(bo); 1781 1782 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1783 out: 1784 1785 /** 1786 * 1787 * Unreserve without putting on LRU to avoid swapping out an 1788 * already swapped buffer. 1789 */ 1790 1791 __ttm_bo_unreserve(bo); 1792 kref_put(&bo->list_kref, ttm_bo_release_list); 1793 return ret; 1794 } 1795 1796 void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1797 { 1798 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1799 ; 1800 } 1801 EXPORT_SYMBOL(ttm_bo_swapout_all); 1802 1803 /** 1804 * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become 1805 * unreserved 1806 * 1807 * @bo: Pointer to buffer 1808 */ 1809 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) 1810 { 1811 int ret; 1812 1813 /* 1814 * In the absense of a wait_unlocked API, 1815 * Use the bo::wu_mutex to avoid triggering livelocks due to 1816 * concurrent use of this function. Note that this use of 1817 * bo::wu_mutex can go away if we change locking order to 1818 * mmap_sem -> bo::reserve. 1819 */ 1820 ret = mutex_lock_interruptible(&bo->wu_mutex); 1821 if (unlikely(ret != 0)) 1822 return -ERESTARTSYS; 1823 if (!ww_mutex_is_locked(&bo->resv->lock)) 1824 goto out_unlock; 1825 ret = __ttm_bo_reserve(bo, true, false, NULL); 1826 if (unlikely(ret != 0)) 1827 goto out_unlock; 1828 __ttm_bo_unreserve(bo); 1829 1830 out_unlock: 1831 mutex_unlock(&bo->wu_mutex); 1832 return ret; 1833 } 1834