1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #define pr_fmt(fmt) "[TTM] " fmt 32 33 #include <drm/ttm/ttm_module.h> 34 #include <drm/ttm/ttm_bo_driver.h> 35 #include <drm/ttm/ttm_placement.h> 36 #include <linux/jiffies.h> 37 #include <linux/slab.h> 38 #include <linux/sched.h> 39 #include <linux/mm.h> 40 #include <linux/file.h> 41 #include <linux/module.h> 42 #include <linux/atomic.h> 43 #include <linux/reservation.h> 44 45 #define TTM_ASSERT_LOCKED(param) 46 #define TTM_DEBUG(fmt, arg...) 47 #define TTM_BO_HASH_ORDER 13 48 49 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); 50 static void ttm_bo_global_kobj_release(struct kobject *kobj); 51 52 static struct attribute ttm_bo_count = { 53 .name = "bo_count", 54 .mode = S_IRUGO 55 }; 56 57 static inline int ttm_mem_type_from_place(const struct ttm_place *place, 58 uint32_t *mem_type) 59 { 60 int pos; 61 62 pos = ffs(place->flags & TTM_PL_MASK_MEM); 63 if (unlikely(!pos)) 64 return -EINVAL; 65 66 *mem_type = pos - 1; 67 return 0; 68 } 69 70 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 71 { 72 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 73 74 pr_err(" has_type: %d\n", man->has_type); 75 pr_err(" use_type: %d\n", man->use_type); 76 pr_err(" flags: 0x%08X\n", man->flags); 77 pr_err(" gpu_offset: 0x%08llX\n", man->gpu_offset); 78 pr_err(" size: %llu\n", man->size); 79 pr_err(" available_caching: 0x%08X\n", man->available_caching); 80 pr_err(" default_caching: 0x%08X\n", man->default_caching); 81 if (mem_type != TTM_PL_SYSTEM) 82 (*man->func->debug)(man, TTM_PFX); 83 } 84 85 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 86 struct ttm_placement *placement) 87 { 88 int i, ret, mem_type; 89 90 pr_err("No space for %p (%lu pages, %luK, %luM)\n", 91 bo, bo->mem.num_pages, bo->mem.size >> 10, 92 bo->mem.size >> 20); 93 for (i = 0; i < placement->num_placement; i++) { 94 ret = ttm_mem_type_from_place(&placement->placement[i], 95 &mem_type); 96 if (ret) 97 return; 98 pr_err(" placement[%d]=0x%08X (%d)\n", 99 i, placement->placement[i].flags, mem_type); 100 ttm_mem_type_debug(bo->bdev, mem_type); 101 } 102 } 103 104 static ssize_t ttm_bo_global_show(struct kobject *kobj, 105 struct attribute *attr, 106 char *buffer) 107 { 108 struct ttm_bo_global *glob = 109 container_of(kobj, struct ttm_bo_global, kobj); 110 111 return snprintf(buffer, PAGE_SIZE, "%lu\n", 112 (unsigned long) atomic_read(&glob->bo_count)); 113 } 114 115 static struct attribute *ttm_bo_global_attrs[] = { 116 &ttm_bo_count, 117 NULL 118 }; 119 120 static const struct sysfs_ops ttm_bo_global_ops = { 121 .show = &ttm_bo_global_show 122 }; 123 124 static struct kobj_type ttm_bo_glob_kobj_type = { 125 .release = &ttm_bo_global_kobj_release, 126 .sysfs_ops = &ttm_bo_global_ops, 127 .default_attrs = ttm_bo_global_attrs 128 }; 129 130 131 static inline uint32_t ttm_bo_type_flags(unsigned type) 132 { 133 return 1 << (type); 134 } 135 136 static void ttm_bo_release_list(struct kref *list_kref) 137 { 138 struct ttm_buffer_object *bo = 139 container_of(list_kref, struct ttm_buffer_object, list_kref); 140 struct ttm_bo_device *bdev = bo->bdev; 141 size_t acc_size = bo->acc_size; 142 143 BUG_ON(atomic_read(&bo->list_kref.refcount)); 144 BUG_ON(atomic_read(&bo->kref.refcount)); 145 BUG_ON(atomic_read(&bo->cpu_writers)); 146 BUG_ON(bo->mem.mm_node != NULL); 147 BUG_ON(!list_empty(&bo->lru)); 148 BUG_ON(!list_empty(&bo->ddestroy)); 149 ttm_tt_destroy(bo->ttm); 150 atomic_dec(&bo->glob->bo_count); 151 fence_put(bo->moving); 152 if (bo->resv == &bo->ttm_resv) 153 reservation_object_fini(&bo->ttm_resv); 154 mutex_destroy(&bo->wu_mutex); 155 if (bo->destroy) 156 bo->destroy(bo); 157 else { 158 kfree(bo); 159 } 160 ttm_mem_global_free(bdev->glob->mem_glob, acc_size); 161 } 162 163 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 164 { 165 struct ttm_bo_device *bdev = bo->bdev; 166 167 lockdep_assert_held(&bo->resv->lock.base); 168 169 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 170 171 BUG_ON(!list_empty(&bo->lru)); 172 173 list_add(&bo->lru, bdev->driver->lru_tail(bo)); 174 kref_get(&bo->list_kref); 175 176 if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { 177 list_add(&bo->swap, bdev->driver->swap_lru_tail(bo)); 178 kref_get(&bo->list_kref); 179 } 180 } 181 } 182 EXPORT_SYMBOL(ttm_bo_add_to_lru); 183 184 int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 185 { 186 struct ttm_bo_device *bdev = bo->bdev; 187 int put_count = 0; 188 189 if (bdev->driver->lru_removal) 190 bdev->driver->lru_removal(bo); 191 192 if (!list_empty(&bo->swap)) { 193 list_del_init(&bo->swap); 194 ++put_count; 195 } 196 if (!list_empty(&bo->lru)) { 197 list_del_init(&bo->lru); 198 ++put_count; 199 } 200 201 return put_count; 202 } 203 204 static void ttm_bo_ref_bug(struct kref *list_kref) 205 { 206 BUG(); 207 } 208 209 void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, 210 bool never_free) 211 { 212 kref_sub(&bo->list_kref, count, 213 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); 214 } 215 216 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) 217 { 218 int put_count; 219 220 spin_lock(&bo->glob->lru_lock); 221 put_count = ttm_bo_del_from_lru(bo); 222 spin_unlock(&bo->glob->lru_lock); 223 ttm_bo_list_ref_sub(bo, put_count, true); 224 } 225 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); 226 227 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) 228 { 229 struct ttm_bo_device *bdev = bo->bdev; 230 int put_count = 0; 231 232 lockdep_assert_held(&bo->resv->lock.base); 233 234 if (bdev->driver->lru_removal) 235 bdev->driver->lru_removal(bo); 236 237 put_count = ttm_bo_del_from_lru(bo); 238 ttm_bo_list_ref_sub(bo, put_count, true); 239 ttm_bo_add_to_lru(bo); 240 } 241 EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); 242 243 struct list_head *ttm_bo_default_lru_tail(struct ttm_buffer_object *bo) 244 { 245 return bo->bdev->man[bo->mem.mem_type].lru.prev; 246 } 247 EXPORT_SYMBOL(ttm_bo_default_lru_tail); 248 249 struct list_head *ttm_bo_default_swap_lru_tail(struct ttm_buffer_object *bo) 250 { 251 return bo->glob->swap_lru.prev; 252 } 253 EXPORT_SYMBOL(ttm_bo_default_swap_lru_tail); 254 255 /* 256 * Call bo->mutex locked. 257 */ 258 static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 259 { 260 struct ttm_bo_device *bdev = bo->bdev; 261 struct ttm_bo_global *glob = bo->glob; 262 int ret = 0; 263 uint32_t page_flags = 0; 264 265 TTM_ASSERT_LOCKED(&bo->mutex); 266 bo->ttm = NULL; 267 268 if (bdev->need_dma32) 269 page_flags |= TTM_PAGE_FLAG_DMA32; 270 271 switch (bo->type) { 272 case ttm_bo_type_device: 273 if (zero_alloc) 274 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 275 case ttm_bo_type_kernel: 276 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 277 page_flags, glob->dummy_read_page); 278 if (unlikely(bo->ttm == NULL)) 279 ret = -ENOMEM; 280 break; 281 case ttm_bo_type_sg: 282 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 283 page_flags | TTM_PAGE_FLAG_SG, 284 glob->dummy_read_page); 285 if (unlikely(bo->ttm == NULL)) { 286 ret = -ENOMEM; 287 break; 288 } 289 bo->ttm->sg = bo->sg; 290 break; 291 default: 292 pr_err("Illegal buffer object type\n"); 293 ret = -EINVAL; 294 break; 295 } 296 297 return ret; 298 } 299 300 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 301 struct ttm_mem_reg *mem, 302 bool evict, bool interruptible, 303 bool no_wait_gpu) 304 { 305 struct ttm_bo_device *bdev = bo->bdev; 306 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 307 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 308 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 309 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 310 int ret = 0; 311 312 if (old_is_pci || new_is_pci || 313 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 314 ret = ttm_mem_io_lock(old_man, true); 315 if (unlikely(ret != 0)) 316 goto out_err; 317 ttm_bo_unmap_virtual_locked(bo); 318 ttm_mem_io_unlock(old_man); 319 } 320 321 /* 322 * Create and bind a ttm if required. 323 */ 324 325 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 326 if (bo->ttm == NULL) { 327 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 328 ret = ttm_bo_add_ttm(bo, zero); 329 if (ret) 330 goto out_err; 331 } 332 333 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 334 if (ret) 335 goto out_err; 336 337 if (mem->mem_type != TTM_PL_SYSTEM) { 338 ret = ttm_tt_bind(bo->ttm, mem); 339 if (ret) 340 goto out_err; 341 } 342 343 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 344 if (bdev->driver->move_notify) 345 bdev->driver->move_notify(bo, mem); 346 bo->mem = *mem; 347 mem->mm_node = NULL; 348 goto moved; 349 } 350 } 351 352 if (bdev->driver->move_notify) 353 bdev->driver->move_notify(bo, mem); 354 355 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 356 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 357 ret = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, mem); 358 else if (bdev->driver->move) 359 ret = bdev->driver->move(bo, evict, interruptible, 360 no_wait_gpu, mem); 361 else 362 ret = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, mem); 363 364 if (ret) { 365 if (bdev->driver->move_notify) { 366 struct ttm_mem_reg tmp_mem = *mem; 367 *mem = bo->mem; 368 bo->mem = tmp_mem; 369 bdev->driver->move_notify(bo, mem); 370 bo->mem = *mem; 371 *mem = tmp_mem; 372 } 373 374 goto out_err; 375 } 376 377 moved: 378 if (bo->evicted) { 379 if (bdev->driver->invalidate_caches) { 380 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 381 if (ret) 382 pr_err("Can not flush read caches\n"); 383 } 384 bo->evicted = false; 385 } 386 387 if (bo->mem.mm_node) { 388 bo->offset = (bo->mem.start << PAGE_SHIFT) + 389 bdev->man[bo->mem.mem_type].gpu_offset; 390 bo->cur_placement = bo->mem.placement; 391 } else 392 bo->offset = 0; 393 394 return 0; 395 396 out_err: 397 new_man = &bdev->man[bo->mem.mem_type]; 398 if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) { 399 ttm_tt_destroy(bo->ttm); 400 bo->ttm = NULL; 401 } 402 403 return ret; 404 } 405 406 /** 407 * Call bo::reserved. 408 * Will release GPU memory type usage on destruction. 409 * This is the place to put in driver specific hooks to release 410 * driver private resources. 411 * Will release the bo::reserved lock. 412 */ 413 414 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 415 { 416 if (bo->bdev->driver->move_notify) 417 bo->bdev->driver->move_notify(bo, NULL); 418 419 ttm_tt_destroy(bo->ttm); 420 bo->ttm = NULL; 421 ttm_bo_mem_put(bo, &bo->mem); 422 423 ww_mutex_unlock (&bo->resv->lock); 424 } 425 426 static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) 427 { 428 struct reservation_object_list *fobj; 429 struct fence *fence; 430 int i; 431 432 fobj = reservation_object_get_list(bo->resv); 433 fence = reservation_object_get_excl(bo->resv); 434 if (fence && !fence->ops->signaled) 435 fence_enable_sw_signaling(fence); 436 437 for (i = 0; fobj && i < fobj->shared_count; ++i) { 438 fence = rcu_dereference_protected(fobj->shared[i], 439 reservation_object_held(bo->resv)); 440 441 if (!fence->ops->signaled) 442 fence_enable_sw_signaling(fence); 443 } 444 } 445 446 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 447 { 448 struct ttm_bo_device *bdev = bo->bdev; 449 struct ttm_bo_global *glob = bo->glob; 450 int put_count; 451 int ret; 452 453 spin_lock(&glob->lru_lock); 454 ret = __ttm_bo_reserve(bo, false, true, NULL); 455 456 if (!ret) { 457 if (!ttm_bo_wait(bo, false, true)) { 458 put_count = ttm_bo_del_from_lru(bo); 459 460 spin_unlock(&glob->lru_lock); 461 ttm_bo_cleanup_memtype_use(bo); 462 463 ttm_bo_list_ref_sub(bo, put_count, true); 464 465 return; 466 } else 467 ttm_bo_flush_all_fences(bo); 468 469 /* 470 * Make NO_EVICT bos immediately available to 471 * shrinkers, now that they are queued for 472 * destruction. 473 */ 474 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { 475 bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; 476 ttm_bo_add_to_lru(bo); 477 } 478 479 __ttm_bo_unreserve(bo); 480 } 481 482 kref_get(&bo->list_kref); 483 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 484 spin_unlock(&glob->lru_lock); 485 486 schedule_delayed_work(&bdev->wq, 487 ((HZ / 100) < 1) ? 1 : HZ / 100); 488 } 489 490 /** 491 * function ttm_bo_cleanup_refs_and_unlock 492 * If bo idle, remove from delayed- and lru lists, and unref. 493 * If not idle, do nothing. 494 * 495 * Must be called with lru_lock and reservation held, this function 496 * will drop both before returning. 497 * 498 * @interruptible Any sleeps should occur interruptibly. 499 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 500 */ 501 502 static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, 503 bool interruptible, 504 bool no_wait_gpu) 505 { 506 struct ttm_bo_global *glob = bo->glob; 507 int put_count; 508 int ret; 509 510 ret = ttm_bo_wait(bo, false, true); 511 512 if (ret && !no_wait_gpu) { 513 long lret; 514 ww_mutex_unlock(&bo->resv->lock); 515 spin_unlock(&glob->lru_lock); 516 517 lret = reservation_object_wait_timeout_rcu(bo->resv, 518 true, 519 interruptible, 520 30 * HZ); 521 522 if (lret < 0) 523 return lret; 524 else if (lret == 0) 525 return -EBUSY; 526 527 spin_lock(&glob->lru_lock); 528 ret = __ttm_bo_reserve(bo, false, true, NULL); 529 530 /* 531 * We raced, and lost, someone else holds the reservation now, 532 * and is probably busy in ttm_bo_cleanup_memtype_use. 533 * 534 * Even if it's not the case, because we finished waiting any 535 * delayed destruction would succeed, so just return success 536 * here. 537 */ 538 if (ret) { 539 spin_unlock(&glob->lru_lock); 540 return 0; 541 } 542 543 /* 544 * remove sync_obj with ttm_bo_wait, the wait should be 545 * finished, and no new wait object should have been added. 546 */ 547 ret = ttm_bo_wait(bo, false, true); 548 WARN_ON(ret); 549 } 550 551 if (ret || unlikely(list_empty(&bo->ddestroy))) { 552 __ttm_bo_unreserve(bo); 553 spin_unlock(&glob->lru_lock); 554 return ret; 555 } 556 557 put_count = ttm_bo_del_from_lru(bo); 558 list_del_init(&bo->ddestroy); 559 ++put_count; 560 561 spin_unlock(&glob->lru_lock); 562 ttm_bo_cleanup_memtype_use(bo); 563 564 ttm_bo_list_ref_sub(bo, put_count, true); 565 566 return 0; 567 } 568 569 /** 570 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 571 * encountered buffers. 572 */ 573 574 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 575 { 576 struct ttm_bo_global *glob = bdev->glob; 577 struct ttm_buffer_object *entry = NULL; 578 int ret = 0; 579 580 spin_lock(&glob->lru_lock); 581 if (list_empty(&bdev->ddestroy)) 582 goto out_unlock; 583 584 entry = list_first_entry(&bdev->ddestroy, 585 struct ttm_buffer_object, ddestroy); 586 kref_get(&entry->list_kref); 587 588 for (;;) { 589 struct ttm_buffer_object *nentry = NULL; 590 591 if (entry->ddestroy.next != &bdev->ddestroy) { 592 nentry = list_first_entry(&entry->ddestroy, 593 struct ttm_buffer_object, ddestroy); 594 kref_get(&nentry->list_kref); 595 } 596 597 ret = __ttm_bo_reserve(entry, false, true, NULL); 598 if (remove_all && ret) { 599 spin_unlock(&glob->lru_lock); 600 ret = __ttm_bo_reserve(entry, false, false, NULL); 601 spin_lock(&glob->lru_lock); 602 } 603 604 if (!ret) 605 ret = ttm_bo_cleanup_refs_and_unlock(entry, false, 606 !remove_all); 607 else 608 spin_unlock(&glob->lru_lock); 609 610 kref_put(&entry->list_kref, ttm_bo_release_list); 611 entry = nentry; 612 613 if (ret || !entry) 614 goto out; 615 616 spin_lock(&glob->lru_lock); 617 if (list_empty(&entry->ddestroy)) 618 break; 619 } 620 621 out_unlock: 622 spin_unlock(&glob->lru_lock); 623 out: 624 if (entry) 625 kref_put(&entry->list_kref, ttm_bo_release_list); 626 return ret; 627 } 628 629 static void ttm_bo_delayed_workqueue(struct work_struct *work) 630 { 631 struct ttm_bo_device *bdev = 632 container_of(work, struct ttm_bo_device, wq.work); 633 634 if (ttm_bo_delayed_delete(bdev, false)) { 635 schedule_delayed_work(&bdev->wq, 636 ((HZ / 100) < 1) ? 1 : HZ / 100); 637 } 638 } 639 640 static void ttm_bo_release(struct kref *kref) 641 { 642 struct ttm_buffer_object *bo = 643 container_of(kref, struct ttm_buffer_object, kref); 644 struct ttm_bo_device *bdev = bo->bdev; 645 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 646 647 drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); 648 ttm_mem_io_lock(man, false); 649 ttm_mem_io_free_vm(bo); 650 ttm_mem_io_unlock(man); 651 ttm_bo_cleanup_refs_or_queue(bo); 652 kref_put(&bo->list_kref, ttm_bo_release_list); 653 } 654 655 void ttm_bo_unref(struct ttm_buffer_object **p_bo) 656 { 657 struct ttm_buffer_object *bo = *p_bo; 658 659 *p_bo = NULL; 660 kref_put(&bo->kref, ttm_bo_release); 661 } 662 EXPORT_SYMBOL(ttm_bo_unref); 663 664 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 665 { 666 return cancel_delayed_work_sync(&bdev->wq); 667 } 668 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 669 670 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 671 { 672 if (resched) 673 schedule_delayed_work(&bdev->wq, 674 ((HZ / 100) < 1) ? 1 : HZ / 100); 675 } 676 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 677 678 static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 679 bool no_wait_gpu) 680 { 681 struct ttm_bo_device *bdev = bo->bdev; 682 struct ttm_mem_reg evict_mem; 683 struct ttm_placement placement; 684 int ret = 0; 685 686 lockdep_assert_held(&bo->resv->lock.base); 687 688 evict_mem = bo->mem; 689 evict_mem.mm_node = NULL; 690 evict_mem.bus.io_reserved_vm = false; 691 evict_mem.bus.io_reserved_count = 0; 692 693 placement.num_placement = 0; 694 placement.num_busy_placement = 0; 695 bdev->driver->evict_flags(bo, &placement); 696 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 697 no_wait_gpu); 698 if (ret) { 699 if (ret != -ERESTARTSYS) { 700 pr_err("Failed to find memory space for buffer 0x%p eviction\n", 701 bo); 702 ttm_bo_mem_space_debug(bo, &placement); 703 } 704 goto out; 705 } 706 707 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 708 no_wait_gpu); 709 if (unlikely(ret)) { 710 if (ret != -ERESTARTSYS) 711 pr_err("Buffer eviction failed\n"); 712 ttm_bo_mem_put(bo, &evict_mem); 713 goto out; 714 } 715 bo->evicted = true; 716 out: 717 return ret; 718 } 719 720 static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 721 uint32_t mem_type, 722 const struct ttm_place *place, 723 bool interruptible, 724 bool no_wait_gpu) 725 { 726 struct ttm_bo_global *glob = bdev->glob; 727 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 728 struct ttm_buffer_object *bo; 729 int ret = -EBUSY, put_count; 730 731 spin_lock(&glob->lru_lock); 732 list_for_each_entry(bo, &man->lru, lru) { 733 ret = __ttm_bo_reserve(bo, false, true, NULL); 734 if (!ret) { 735 if (place && (place->fpfn || place->lpfn)) { 736 /* Don't evict this BO if it's outside of the 737 * requested placement range 738 */ 739 if (place->fpfn >= (bo->mem.start + bo->mem.size) || 740 (place->lpfn && place->lpfn <= bo->mem.start)) { 741 __ttm_bo_unreserve(bo); 742 ret = -EBUSY; 743 continue; 744 } 745 } 746 747 break; 748 } 749 } 750 751 if (ret) { 752 spin_unlock(&glob->lru_lock); 753 return ret; 754 } 755 756 kref_get(&bo->list_kref); 757 758 if (!list_empty(&bo->ddestroy)) { 759 ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, 760 no_wait_gpu); 761 kref_put(&bo->list_kref, ttm_bo_release_list); 762 return ret; 763 } 764 765 put_count = ttm_bo_del_from_lru(bo); 766 spin_unlock(&glob->lru_lock); 767 768 BUG_ON(ret != 0); 769 770 ttm_bo_list_ref_sub(bo, put_count, true); 771 772 ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); 773 ttm_bo_unreserve(bo); 774 775 kref_put(&bo->list_kref, ttm_bo_release_list); 776 return ret; 777 } 778 779 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 780 { 781 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 782 783 if (mem->mm_node) 784 (*man->func->put_node)(man, mem); 785 } 786 EXPORT_SYMBOL(ttm_bo_mem_put); 787 788 /** 789 * Add the last move fence to the BO and reserve a new shared slot. 790 */ 791 static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, 792 struct ttm_mem_type_manager *man, 793 struct ttm_mem_reg *mem) 794 { 795 struct fence *fence; 796 int ret; 797 798 spin_lock(&man->move_lock); 799 fence = fence_get(man->move); 800 spin_unlock(&man->move_lock); 801 802 if (fence) { 803 reservation_object_add_shared_fence(bo->resv, fence); 804 805 ret = reservation_object_reserve_shared(bo->resv); 806 if (unlikely(ret)) 807 return ret; 808 809 fence_put(bo->moving); 810 bo->moving = fence; 811 } 812 813 return 0; 814 } 815 816 /** 817 * Repeatedly evict memory from the LRU for @mem_type until we create enough 818 * space, or we've evicted everything and there isn't enough space. 819 */ 820 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 821 uint32_t mem_type, 822 const struct ttm_place *place, 823 struct ttm_mem_reg *mem, 824 bool interruptible, 825 bool no_wait_gpu) 826 { 827 struct ttm_bo_device *bdev = bo->bdev; 828 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 829 int ret; 830 831 do { 832 ret = (*man->func->get_node)(man, bo, place, mem); 833 if (unlikely(ret != 0)) 834 return ret; 835 if (mem->mm_node) 836 break; 837 ret = ttm_mem_evict_first(bdev, mem_type, place, 838 interruptible, no_wait_gpu); 839 if (unlikely(ret != 0)) 840 return ret; 841 } while (1); 842 mem->mem_type = mem_type; 843 return ttm_bo_add_move_fence(bo, man, mem); 844 } 845 846 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 847 uint32_t cur_placement, 848 uint32_t proposed_placement) 849 { 850 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 851 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 852 853 /** 854 * Keep current caching if possible. 855 */ 856 857 if ((cur_placement & caching) != 0) 858 result |= (cur_placement & caching); 859 else if ((man->default_caching & caching) != 0) 860 result |= man->default_caching; 861 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 862 result |= TTM_PL_FLAG_CACHED; 863 else if ((TTM_PL_FLAG_WC & caching) != 0) 864 result |= TTM_PL_FLAG_WC; 865 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 866 result |= TTM_PL_FLAG_UNCACHED; 867 868 return result; 869 } 870 871 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 872 uint32_t mem_type, 873 const struct ttm_place *place, 874 uint32_t *masked_placement) 875 { 876 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 877 878 if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0) 879 return false; 880 881 if ((place->flags & man->available_caching) == 0) 882 return false; 883 884 cur_flags |= (place->flags & man->available_caching); 885 886 *masked_placement = cur_flags; 887 return true; 888 } 889 890 /** 891 * Creates space for memory region @mem according to its type. 892 * 893 * This function first searches for free space in compatible memory types in 894 * the priority order defined by the driver. If free space isn't found, then 895 * ttm_bo_mem_force_space is attempted in priority order to evict and find 896 * space. 897 */ 898 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 899 struct ttm_placement *placement, 900 struct ttm_mem_reg *mem, 901 bool interruptible, 902 bool no_wait_gpu) 903 { 904 struct ttm_bo_device *bdev = bo->bdev; 905 struct ttm_mem_type_manager *man; 906 uint32_t mem_type = TTM_PL_SYSTEM; 907 uint32_t cur_flags = 0; 908 bool type_found = false; 909 bool type_ok = false; 910 bool has_erestartsys = false; 911 int i, ret; 912 913 ret = reservation_object_reserve_shared(bo->resv); 914 if (unlikely(ret)) 915 return ret; 916 917 mem->mm_node = NULL; 918 for (i = 0; i < placement->num_placement; ++i) { 919 const struct ttm_place *place = &placement->placement[i]; 920 921 ret = ttm_mem_type_from_place(place, &mem_type); 922 if (ret) 923 return ret; 924 man = &bdev->man[mem_type]; 925 if (!man->has_type || !man->use_type) 926 continue; 927 928 type_ok = ttm_bo_mt_compatible(man, mem_type, place, 929 &cur_flags); 930 931 if (!type_ok) 932 continue; 933 934 type_found = true; 935 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 936 cur_flags); 937 /* 938 * Use the access and other non-mapping-related flag bits from 939 * the memory placement flags to the current flags 940 */ 941 ttm_flag_masked(&cur_flags, place->flags, 942 ~TTM_PL_MASK_MEMTYPE); 943 944 if (mem_type == TTM_PL_SYSTEM) 945 break; 946 947 ret = (*man->func->get_node)(man, bo, place, mem); 948 if (unlikely(ret)) 949 return ret; 950 951 if (mem->mm_node) { 952 ret = ttm_bo_add_move_fence(bo, man, mem); 953 if (unlikely(ret)) { 954 (*man->func->put_node)(man, mem); 955 return ret; 956 } 957 break; 958 } 959 } 960 961 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 962 mem->mem_type = mem_type; 963 mem->placement = cur_flags; 964 return 0; 965 } 966 967 for (i = 0; i < placement->num_busy_placement; ++i) { 968 const struct ttm_place *place = &placement->busy_placement[i]; 969 970 ret = ttm_mem_type_from_place(place, &mem_type); 971 if (ret) 972 return ret; 973 man = &bdev->man[mem_type]; 974 if (!man->has_type || !man->use_type) 975 continue; 976 if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags)) 977 continue; 978 979 type_found = true; 980 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 981 cur_flags); 982 /* 983 * Use the access and other non-mapping-related flag bits from 984 * the memory placement flags to the current flags 985 */ 986 ttm_flag_masked(&cur_flags, place->flags, 987 ~TTM_PL_MASK_MEMTYPE); 988 989 if (mem_type == TTM_PL_SYSTEM) { 990 mem->mem_type = mem_type; 991 mem->placement = cur_flags; 992 mem->mm_node = NULL; 993 return 0; 994 } 995 996 ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, 997 interruptible, no_wait_gpu); 998 if (ret == 0 && mem->mm_node) { 999 mem->placement = cur_flags; 1000 return 0; 1001 } 1002 if (ret == -ERESTARTSYS) 1003 has_erestartsys = true; 1004 } 1005 1006 if (!type_found) { 1007 printk(KERN_ERR TTM_PFX "No compatible memory type found.\n"); 1008 return -EINVAL; 1009 } 1010 1011 return (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; 1012 } 1013 EXPORT_SYMBOL(ttm_bo_mem_space); 1014 1015 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 1016 struct ttm_placement *placement, 1017 bool interruptible, 1018 bool no_wait_gpu) 1019 { 1020 int ret = 0; 1021 struct ttm_mem_reg mem; 1022 1023 lockdep_assert_held(&bo->resv->lock.base); 1024 1025 mem.num_pages = bo->num_pages; 1026 mem.size = mem.num_pages << PAGE_SHIFT; 1027 mem.page_alignment = bo->mem.page_alignment; 1028 mem.bus.io_reserved_vm = false; 1029 mem.bus.io_reserved_count = 0; 1030 /* 1031 * Determine where to move the buffer. 1032 */ 1033 ret = ttm_bo_mem_space(bo, placement, &mem, 1034 interruptible, no_wait_gpu); 1035 if (ret) 1036 goto out_unlock; 1037 ret = ttm_bo_handle_move_mem(bo, &mem, false, 1038 interruptible, no_wait_gpu); 1039 out_unlock: 1040 if (ret && mem.mm_node) 1041 ttm_bo_mem_put(bo, &mem); 1042 return ret; 1043 } 1044 1045 bool ttm_bo_mem_compat(struct ttm_placement *placement, 1046 struct ttm_mem_reg *mem, 1047 uint32_t *new_flags) 1048 { 1049 int i; 1050 1051 for (i = 0; i < placement->num_placement; i++) { 1052 const struct ttm_place *heap = &placement->placement[i]; 1053 if (mem->mm_node && 1054 (mem->start < heap->fpfn || 1055 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) 1056 continue; 1057 1058 *new_flags = heap->flags; 1059 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1060 (*new_flags & mem->placement & TTM_PL_MASK_MEM)) 1061 return true; 1062 } 1063 1064 for (i = 0; i < placement->num_busy_placement; i++) { 1065 const struct ttm_place *heap = &placement->busy_placement[i]; 1066 if (mem->mm_node && 1067 (mem->start < heap->fpfn || 1068 (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) 1069 continue; 1070 1071 *new_flags = heap->flags; 1072 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1073 (*new_flags & mem->placement & TTM_PL_MASK_MEM)) 1074 return true; 1075 } 1076 1077 return false; 1078 } 1079 EXPORT_SYMBOL(ttm_bo_mem_compat); 1080 1081 int ttm_bo_validate(struct ttm_buffer_object *bo, 1082 struct ttm_placement *placement, 1083 bool interruptible, 1084 bool no_wait_gpu) 1085 { 1086 int ret; 1087 uint32_t new_flags; 1088 1089 lockdep_assert_held(&bo->resv->lock.base); 1090 /* 1091 * Check whether we need to move buffer. 1092 */ 1093 if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { 1094 ret = ttm_bo_move_buffer(bo, placement, interruptible, 1095 no_wait_gpu); 1096 if (ret) 1097 return ret; 1098 } else { 1099 /* 1100 * Use the access and other non-mapping-related flag bits from 1101 * the compatible memory placement flags to the active flags 1102 */ 1103 ttm_flag_masked(&bo->mem.placement, new_flags, 1104 ~TTM_PL_MASK_MEMTYPE); 1105 } 1106 /* 1107 * We might need to add a TTM. 1108 */ 1109 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1110 ret = ttm_bo_add_ttm(bo, true); 1111 if (ret) 1112 return ret; 1113 } 1114 return 0; 1115 } 1116 EXPORT_SYMBOL(ttm_bo_validate); 1117 1118 int ttm_bo_init(struct ttm_bo_device *bdev, 1119 struct ttm_buffer_object *bo, 1120 unsigned long size, 1121 enum ttm_bo_type type, 1122 struct ttm_placement *placement, 1123 uint32_t page_alignment, 1124 bool interruptible, 1125 struct file *persistent_swap_storage, 1126 size_t acc_size, 1127 struct sg_table *sg, 1128 struct reservation_object *resv, 1129 void (*destroy) (struct ttm_buffer_object *)) 1130 { 1131 int ret = 0; 1132 unsigned long num_pages; 1133 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1134 bool locked; 1135 1136 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1137 if (ret) { 1138 pr_err("Out of kernel memory\n"); 1139 if (destroy) 1140 (*destroy)(bo); 1141 else 1142 kfree(bo); 1143 return -ENOMEM; 1144 } 1145 1146 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1147 if (num_pages == 0) { 1148 pr_err("Illegal buffer object size\n"); 1149 if (destroy) 1150 (*destroy)(bo); 1151 else 1152 kfree(bo); 1153 ttm_mem_global_free(mem_glob, acc_size); 1154 return -EINVAL; 1155 } 1156 bo->destroy = destroy; 1157 1158 kref_init(&bo->kref); 1159 kref_init(&bo->list_kref); 1160 atomic_set(&bo->cpu_writers, 0); 1161 INIT_LIST_HEAD(&bo->lru); 1162 INIT_LIST_HEAD(&bo->ddestroy); 1163 INIT_LIST_HEAD(&bo->swap); 1164 INIT_LIST_HEAD(&bo->io_reserve_lru); 1165 mutex_init(&bo->wu_mutex); 1166 bo->bdev = bdev; 1167 bo->glob = bdev->glob; 1168 bo->type = type; 1169 bo->num_pages = num_pages; 1170 bo->mem.size = num_pages << PAGE_SHIFT; 1171 bo->mem.mem_type = TTM_PL_SYSTEM; 1172 bo->mem.num_pages = bo->num_pages; 1173 bo->mem.mm_node = NULL; 1174 bo->mem.page_alignment = page_alignment; 1175 bo->mem.bus.io_reserved_vm = false; 1176 bo->mem.bus.io_reserved_count = 0; 1177 bo->moving = NULL; 1178 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1179 bo->persistent_swap_storage = persistent_swap_storage; 1180 bo->acc_size = acc_size; 1181 bo->sg = sg; 1182 if (resv) { 1183 bo->resv = resv; 1184 lockdep_assert_held(&bo->resv->lock.base); 1185 } else { 1186 bo->resv = &bo->ttm_resv; 1187 reservation_object_init(&bo->ttm_resv); 1188 } 1189 atomic_inc(&bo->glob->bo_count); 1190 drm_vma_node_reset(&bo->vma_node); 1191 1192 /* 1193 * For ttm_bo_type_device buffers, allocate 1194 * address space from the device. 1195 */ 1196 if (bo->type == ttm_bo_type_device || 1197 bo->type == ttm_bo_type_sg) 1198 ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, 1199 bo->mem.num_pages); 1200 1201 /* passed reservation objects should already be locked, 1202 * since otherwise lockdep will be angered in radeon. 1203 */ 1204 if (!resv) { 1205 locked = ww_mutex_trylock(&bo->resv->lock); 1206 WARN_ON(!locked); 1207 } 1208 1209 if (likely(!ret)) 1210 ret = ttm_bo_validate(bo, placement, interruptible, false); 1211 1212 if (!resv) { 1213 ttm_bo_unreserve(bo); 1214 1215 } else if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 1216 spin_lock(&bo->glob->lru_lock); 1217 ttm_bo_add_to_lru(bo); 1218 spin_unlock(&bo->glob->lru_lock); 1219 } 1220 1221 if (unlikely(ret)) 1222 ttm_bo_unref(&bo); 1223 1224 return ret; 1225 } 1226 EXPORT_SYMBOL(ttm_bo_init); 1227 1228 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1229 unsigned long bo_size, 1230 unsigned struct_size) 1231 { 1232 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1233 size_t size = 0; 1234 1235 size += ttm_round_pot(struct_size); 1236 size += ttm_round_pot(npages * sizeof(void *)); 1237 size += ttm_round_pot(sizeof(struct ttm_tt)); 1238 return size; 1239 } 1240 EXPORT_SYMBOL(ttm_bo_acc_size); 1241 1242 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1243 unsigned long bo_size, 1244 unsigned struct_size) 1245 { 1246 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1247 size_t size = 0; 1248 1249 size += ttm_round_pot(struct_size); 1250 size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t))); 1251 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1252 return size; 1253 } 1254 EXPORT_SYMBOL(ttm_bo_dma_acc_size); 1255 1256 int ttm_bo_create(struct ttm_bo_device *bdev, 1257 unsigned long size, 1258 enum ttm_bo_type type, 1259 struct ttm_placement *placement, 1260 uint32_t page_alignment, 1261 bool interruptible, 1262 struct file *persistent_swap_storage, 1263 struct ttm_buffer_object **p_bo) 1264 { 1265 struct ttm_buffer_object *bo; 1266 size_t acc_size; 1267 int ret; 1268 1269 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1270 if (unlikely(bo == NULL)) 1271 return -ENOMEM; 1272 1273 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1274 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1275 interruptible, persistent_swap_storage, acc_size, 1276 NULL, NULL, NULL); 1277 if (likely(ret == 0)) 1278 *p_bo = bo; 1279 1280 return ret; 1281 } 1282 EXPORT_SYMBOL(ttm_bo_create); 1283 1284 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1285 unsigned mem_type, bool allow_errors) 1286 { 1287 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1288 struct ttm_bo_global *glob = bdev->glob; 1289 struct fence *fence; 1290 int ret; 1291 1292 /* 1293 * Can't use standard list traversal since we're unlocking. 1294 */ 1295 1296 spin_lock(&glob->lru_lock); 1297 while (!list_empty(&man->lru)) { 1298 spin_unlock(&glob->lru_lock); 1299 ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false); 1300 if (ret) { 1301 if (allow_errors) { 1302 return ret; 1303 } else { 1304 pr_err("Cleanup eviction failed\n"); 1305 } 1306 } 1307 spin_lock(&glob->lru_lock); 1308 } 1309 spin_unlock(&glob->lru_lock); 1310 1311 spin_lock(&man->move_lock); 1312 fence = fence_get(man->move); 1313 spin_unlock(&man->move_lock); 1314 1315 if (fence) { 1316 ret = fence_wait(fence, false); 1317 fence_put(fence); 1318 if (ret) { 1319 if (allow_errors) { 1320 return ret; 1321 } else { 1322 pr_err("Cleanup eviction failed\n"); 1323 } 1324 } 1325 } 1326 1327 return 0; 1328 } 1329 1330 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1331 { 1332 struct ttm_mem_type_manager *man; 1333 int ret = -EINVAL; 1334 1335 if (mem_type >= TTM_NUM_MEM_TYPES) { 1336 pr_err("Illegal memory type %d\n", mem_type); 1337 return ret; 1338 } 1339 man = &bdev->man[mem_type]; 1340 1341 if (!man->has_type) { 1342 pr_err("Trying to take down uninitialized memory manager type %u\n", 1343 mem_type); 1344 return ret; 1345 } 1346 fence_put(man->move); 1347 1348 man->use_type = false; 1349 man->has_type = false; 1350 1351 ret = 0; 1352 if (mem_type > 0) { 1353 ttm_bo_force_list_clean(bdev, mem_type, false); 1354 1355 ret = (*man->func->takedown)(man); 1356 } 1357 1358 return ret; 1359 } 1360 EXPORT_SYMBOL(ttm_bo_clean_mm); 1361 1362 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1363 { 1364 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1365 1366 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1367 pr_err("Illegal memory manager memory type %u\n", mem_type); 1368 return -EINVAL; 1369 } 1370 1371 if (!man->has_type) { 1372 pr_err("Memory type %u has not been initialized\n", mem_type); 1373 return 0; 1374 } 1375 1376 return ttm_bo_force_list_clean(bdev, mem_type, true); 1377 } 1378 EXPORT_SYMBOL(ttm_bo_evict_mm); 1379 1380 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1381 unsigned long p_size) 1382 { 1383 int ret = -EINVAL; 1384 struct ttm_mem_type_manager *man; 1385 1386 BUG_ON(type >= TTM_NUM_MEM_TYPES); 1387 man = &bdev->man[type]; 1388 BUG_ON(man->has_type); 1389 man->io_reserve_fastpath = true; 1390 man->use_io_reserve_lru = false; 1391 mutex_init(&man->io_reserve_mutex); 1392 spin_lock_init(&man->move_lock); 1393 INIT_LIST_HEAD(&man->io_reserve_lru); 1394 1395 ret = bdev->driver->init_mem_type(bdev, type, man); 1396 if (ret) 1397 return ret; 1398 man->bdev = bdev; 1399 1400 ret = 0; 1401 if (type != TTM_PL_SYSTEM) { 1402 ret = (*man->func->init)(man, p_size); 1403 if (ret) 1404 return ret; 1405 } 1406 man->has_type = true; 1407 man->use_type = true; 1408 man->size = p_size; 1409 1410 INIT_LIST_HEAD(&man->lru); 1411 man->move = NULL; 1412 1413 return 0; 1414 } 1415 EXPORT_SYMBOL(ttm_bo_init_mm); 1416 1417 static void ttm_bo_global_kobj_release(struct kobject *kobj) 1418 { 1419 struct ttm_bo_global *glob = 1420 container_of(kobj, struct ttm_bo_global, kobj); 1421 1422 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1423 __free_page(glob->dummy_read_page); 1424 kfree(glob); 1425 } 1426 1427 void ttm_bo_global_release(struct drm_global_reference *ref) 1428 { 1429 struct ttm_bo_global *glob = ref->object; 1430 1431 kobject_del(&glob->kobj); 1432 kobject_put(&glob->kobj); 1433 } 1434 EXPORT_SYMBOL(ttm_bo_global_release); 1435 1436 int ttm_bo_global_init(struct drm_global_reference *ref) 1437 { 1438 struct ttm_bo_global_ref *bo_ref = 1439 container_of(ref, struct ttm_bo_global_ref, ref); 1440 struct ttm_bo_global *glob = ref->object; 1441 int ret; 1442 1443 mutex_init(&glob->device_list_mutex); 1444 spin_lock_init(&glob->lru_lock); 1445 glob->mem_glob = bo_ref->mem_glob; 1446 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); 1447 1448 if (unlikely(glob->dummy_read_page == NULL)) { 1449 ret = -ENOMEM; 1450 goto out_no_drp; 1451 } 1452 1453 INIT_LIST_HEAD(&glob->swap_lru); 1454 INIT_LIST_HEAD(&glob->device_list); 1455 1456 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1457 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1458 if (unlikely(ret != 0)) { 1459 pr_err("Could not register buffer object swapout\n"); 1460 goto out_no_shrink; 1461 } 1462 1463 atomic_set(&glob->bo_count, 0); 1464 1465 ret = kobject_init_and_add( 1466 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); 1467 if (unlikely(ret != 0)) 1468 kobject_put(&glob->kobj); 1469 return ret; 1470 out_no_shrink: 1471 __free_page(glob->dummy_read_page); 1472 out_no_drp: 1473 kfree(glob); 1474 return ret; 1475 } 1476 EXPORT_SYMBOL(ttm_bo_global_init); 1477 1478 1479 int ttm_bo_device_release(struct ttm_bo_device *bdev) 1480 { 1481 int ret = 0; 1482 unsigned i = TTM_NUM_MEM_TYPES; 1483 struct ttm_mem_type_manager *man; 1484 struct ttm_bo_global *glob = bdev->glob; 1485 1486 while (i--) { 1487 man = &bdev->man[i]; 1488 if (man->has_type) { 1489 man->use_type = false; 1490 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1491 ret = -EBUSY; 1492 pr_err("DRM memory manager type %d is not clean\n", 1493 i); 1494 } 1495 man->has_type = false; 1496 } 1497 } 1498 1499 mutex_lock(&glob->device_list_mutex); 1500 list_del(&bdev->device_list); 1501 mutex_unlock(&glob->device_list_mutex); 1502 1503 cancel_delayed_work_sync(&bdev->wq); 1504 1505 while (ttm_bo_delayed_delete(bdev, true)) 1506 ; 1507 1508 spin_lock(&glob->lru_lock); 1509 if (list_empty(&bdev->ddestroy)) 1510 TTM_DEBUG("Delayed destroy list was clean\n"); 1511 1512 if (list_empty(&bdev->man[0].lru)) 1513 TTM_DEBUG("Swap list was clean\n"); 1514 spin_unlock(&glob->lru_lock); 1515 1516 drm_vma_offset_manager_destroy(&bdev->vma_manager); 1517 1518 return ret; 1519 } 1520 EXPORT_SYMBOL(ttm_bo_device_release); 1521 1522 int ttm_bo_device_init(struct ttm_bo_device *bdev, 1523 struct ttm_bo_global *glob, 1524 struct ttm_bo_driver *driver, 1525 struct address_space *mapping, 1526 uint64_t file_page_offset, 1527 bool need_dma32) 1528 { 1529 int ret = -EINVAL; 1530 1531 bdev->driver = driver; 1532 1533 memset(bdev->man, 0, sizeof(bdev->man)); 1534 1535 /* 1536 * Initialize the system memory buffer type. 1537 * Other types need to be driver / IOCTL initialized. 1538 */ 1539 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1540 if (unlikely(ret != 0)) 1541 goto out_no_sys; 1542 1543 drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, 1544 0x10000000); 1545 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); 1546 INIT_LIST_HEAD(&bdev->ddestroy); 1547 bdev->dev_mapping = mapping; 1548 bdev->glob = glob; 1549 bdev->need_dma32 = need_dma32; 1550 mutex_lock(&glob->device_list_mutex); 1551 list_add_tail(&bdev->device_list, &glob->device_list); 1552 mutex_unlock(&glob->device_list_mutex); 1553 1554 return 0; 1555 out_no_sys: 1556 return ret; 1557 } 1558 EXPORT_SYMBOL(ttm_bo_device_init); 1559 1560 /* 1561 * buffer object vm functions. 1562 */ 1563 1564 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1565 { 1566 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1567 1568 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1569 if (mem->mem_type == TTM_PL_SYSTEM) 1570 return false; 1571 1572 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1573 return false; 1574 1575 if (mem->placement & TTM_PL_FLAG_CACHED) 1576 return false; 1577 } 1578 return true; 1579 } 1580 1581 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1582 { 1583 struct ttm_bo_device *bdev = bo->bdev; 1584 1585 drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); 1586 ttm_mem_io_free_vm(bo); 1587 } 1588 1589 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1590 { 1591 struct ttm_bo_device *bdev = bo->bdev; 1592 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1593 1594 ttm_mem_io_lock(man, false); 1595 ttm_bo_unmap_virtual_locked(bo); 1596 ttm_mem_io_unlock(man); 1597 } 1598 1599 1600 EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1601 1602 int ttm_bo_wait(struct ttm_buffer_object *bo, 1603 bool interruptible, bool no_wait) 1604 { 1605 long timeout = no_wait ? 0 : 15 * HZ; 1606 1607 timeout = reservation_object_wait_timeout_rcu(bo->resv, true, 1608 interruptible, timeout); 1609 if (timeout < 0) 1610 return timeout; 1611 1612 if (timeout == 0) 1613 return -EBUSY; 1614 1615 reservation_object_add_excl_fence(bo->resv, NULL); 1616 return 0; 1617 } 1618 EXPORT_SYMBOL(ttm_bo_wait); 1619 1620 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1621 { 1622 int ret = 0; 1623 1624 /* 1625 * Using ttm_bo_reserve makes sure the lru lists are updated. 1626 */ 1627 1628 ret = ttm_bo_reserve(bo, true, no_wait, NULL); 1629 if (unlikely(ret != 0)) 1630 return ret; 1631 ret = ttm_bo_wait(bo, true, no_wait); 1632 if (likely(ret == 0)) 1633 atomic_inc(&bo->cpu_writers); 1634 ttm_bo_unreserve(bo); 1635 return ret; 1636 } 1637 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); 1638 1639 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1640 { 1641 atomic_dec(&bo->cpu_writers); 1642 } 1643 EXPORT_SYMBOL(ttm_bo_synccpu_write_release); 1644 1645 /** 1646 * A buffer object shrink method that tries to swap out the first 1647 * buffer object on the bo_global::swap_lru list. 1648 */ 1649 1650 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1651 { 1652 struct ttm_bo_global *glob = 1653 container_of(shrink, struct ttm_bo_global, shrink); 1654 struct ttm_buffer_object *bo; 1655 int ret = -EBUSY; 1656 int put_count; 1657 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); 1658 1659 spin_lock(&glob->lru_lock); 1660 list_for_each_entry(bo, &glob->swap_lru, swap) { 1661 ret = __ttm_bo_reserve(bo, false, true, NULL); 1662 if (!ret) 1663 break; 1664 } 1665 1666 if (ret) { 1667 spin_unlock(&glob->lru_lock); 1668 return ret; 1669 } 1670 1671 kref_get(&bo->list_kref); 1672 1673 if (!list_empty(&bo->ddestroy)) { 1674 ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); 1675 kref_put(&bo->list_kref, ttm_bo_release_list); 1676 return ret; 1677 } 1678 1679 put_count = ttm_bo_del_from_lru(bo); 1680 spin_unlock(&glob->lru_lock); 1681 1682 ttm_bo_list_ref_sub(bo, put_count, true); 1683 1684 /** 1685 * Move to system cached 1686 */ 1687 1688 if ((bo->mem.placement & swap_placement) != swap_placement) { 1689 struct ttm_mem_reg evict_mem; 1690 1691 evict_mem = bo->mem; 1692 evict_mem.mm_node = NULL; 1693 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1694 evict_mem.mem_type = TTM_PL_SYSTEM; 1695 1696 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1697 false, false); 1698 if (unlikely(ret != 0)) 1699 goto out; 1700 } 1701 1702 /** 1703 * Make sure BO is idle. 1704 */ 1705 1706 ret = ttm_bo_wait(bo, false, false); 1707 if (unlikely(ret != 0)) 1708 goto out; 1709 1710 ttm_bo_unmap_virtual(bo); 1711 1712 /** 1713 * Swap out. Buffer will be swapped in again as soon as 1714 * anyone tries to access a ttm page. 1715 */ 1716 1717 if (bo->bdev->driver->swap_notify) 1718 bo->bdev->driver->swap_notify(bo); 1719 1720 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1721 out: 1722 1723 /** 1724 * 1725 * Unreserve without putting on LRU to avoid swapping out an 1726 * already swapped buffer. 1727 */ 1728 1729 __ttm_bo_unreserve(bo); 1730 kref_put(&bo->list_kref, ttm_bo_release_list); 1731 return ret; 1732 } 1733 1734 void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1735 { 1736 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1737 ; 1738 } 1739 EXPORT_SYMBOL(ttm_bo_swapout_all); 1740 1741 /** 1742 * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become 1743 * unreserved 1744 * 1745 * @bo: Pointer to buffer 1746 */ 1747 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) 1748 { 1749 int ret; 1750 1751 /* 1752 * In the absense of a wait_unlocked API, 1753 * Use the bo::wu_mutex to avoid triggering livelocks due to 1754 * concurrent use of this function. Note that this use of 1755 * bo::wu_mutex can go away if we change locking order to 1756 * mmap_sem -> bo::reserve. 1757 */ 1758 ret = mutex_lock_interruptible(&bo->wu_mutex); 1759 if (unlikely(ret != 0)) 1760 return -ERESTARTSYS; 1761 if (!ww_mutex_is_locked(&bo->resv->lock)) 1762 goto out_unlock; 1763 ret = __ttm_bo_reserve(bo, true, false, NULL); 1764 if (unlikely(ret != 0)) 1765 goto out_unlock; 1766 __ttm_bo_unreserve(bo); 1767 1768 out_unlock: 1769 mutex_unlock(&bo->wu_mutex); 1770 return ret; 1771 } 1772