1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #define pr_fmt(fmt) "[TTM] " fmt 32 33 #include <drm/ttm/ttm_module.h> 34 #include <drm/ttm/ttm_bo_driver.h> 35 #include <drm/ttm/ttm_placement.h> 36 #include <linux/jiffies.h> 37 #include <linux/slab.h> 38 #include <linux/sched.h> 39 #include <linux/mm.h> 40 #include <linux/file.h> 41 #include <linux/module.h> 42 #include <linux/atomic.h> 43 44 #define TTM_ASSERT_LOCKED(param) 45 #define TTM_DEBUG(fmt, arg...) 46 #define TTM_BO_HASH_ORDER 13 47 48 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); 49 static void ttm_bo_global_kobj_release(struct kobject *kobj); 50 51 static struct attribute ttm_bo_count = { 52 .name = "bo_count", 53 .mode = S_IRUGO 54 }; 55 56 static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type) 57 { 58 int i; 59 60 for (i = 0; i <= TTM_PL_PRIV5; i++) 61 if (flags & (1 << i)) { 62 *mem_type = i; 63 return 0; 64 } 65 return -EINVAL; 66 } 67 68 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 69 { 70 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 71 72 pr_err(" has_type: %d\n", man->has_type); 73 pr_err(" use_type: %d\n", man->use_type); 74 pr_err(" flags: 0x%08X\n", man->flags); 75 pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); 76 pr_err(" size: %llu\n", man->size); 77 pr_err(" available_caching: 0x%08X\n", man->available_caching); 78 pr_err(" default_caching: 0x%08X\n", man->default_caching); 79 if (mem_type != TTM_PL_SYSTEM) 80 (*man->func->debug)(man, TTM_PFX); 81 } 82 83 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 84 struct ttm_placement *placement) 85 { 86 int i, ret, mem_type; 87 88 pr_err("No space for %p (%lu pages, %luK, %luM)\n", 89 bo, bo->mem.num_pages, bo->mem.size >> 10, 90 bo->mem.size >> 20); 91 for (i = 0; i < placement->num_placement; i++) { 92 ret = ttm_mem_type_from_flags(placement->placement[i], 93 &mem_type); 94 if (ret) 95 return; 96 pr_err(" placement[%d]=0x%08X (%d)\n", 97 i, placement->placement[i], mem_type); 98 ttm_mem_type_debug(bo->bdev, mem_type); 99 } 100 } 101 102 static ssize_t ttm_bo_global_show(struct kobject *kobj, 103 struct attribute *attr, 104 char *buffer) 105 { 106 struct ttm_bo_global *glob = 107 container_of(kobj, struct ttm_bo_global, kobj); 108 109 return snprintf(buffer, PAGE_SIZE, "%lu\n", 110 (unsigned long) atomic_read(&glob->bo_count)); 111 } 112 113 static struct attribute *ttm_bo_global_attrs[] = { 114 &ttm_bo_count, 115 NULL 116 }; 117 118 static const struct sysfs_ops ttm_bo_global_ops = { 119 .show = &ttm_bo_global_show 120 }; 121 122 static struct kobj_type ttm_bo_glob_kobj_type = { 123 .release = &ttm_bo_global_kobj_release, 124 .sysfs_ops = &ttm_bo_global_ops, 125 .default_attrs = ttm_bo_global_attrs 126 }; 127 128 129 static inline uint32_t ttm_bo_type_flags(unsigned type) 130 { 131 return 1 << (type); 132 } 133 134 static void ttm_bo_release_list(struct kref *list_kref) 135 { 136 struct ttm_buffer_object *bo = 137 container_of(list_kref, struct ttm_buffer_object, list_kref); 138 struct ttm_bo_device *bdev = bo->bdev; 139 size_t acc_size = bo->acc_size; 140 141 BUG_ON(atomic_read(&bo->list_kref.refcount)); 142 BUG_ON(atomic_read(&bo->kref.refcount)); 143 BUG_ON(atomic_read(&bo->cpu_writers)); 144 BUG_ON(bo->sync_obj != NULL); 145 BUG_ON(bo->mem.mm_node != NULL); 146 BUG_ON(!list_empty(&bo->lru)); 147 BUG_ON(!list_empty(&bo->ddestroy)); 148 149 if (bo->ttm) 150 ttm_tt_destroy(bo->ttm); 151 atomic_dec(&bo->glob->bo_count); 152 if (bo->resv == &bo->ttm_resv) 153 reservation_object_fini(&bo->ttm_resv); 154 mutex_destroy(&bo->wu_mutex); 155 if (bo->destroy) 156 bo->destroy(bo); 157 else { 158 kfree(bo); 159 } 160 ttm_mem_global_free(bdev->glob->mem_glob, acc_size); 161 } 162 163 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 164 { 165 struct ttm_bo_device *bdev = bo->bdev; 166 struct ttm_mem_type_manager *man; 167 168 lockdep_assert_held(&bo->resv->lock.base); 169 170 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 171 172 BUG_ON(!list_empty(&bo->lru)); 173 174 man = &bdev->man[bo->mem.mem_type]; 175 list_add_tail(&bo->lru, &man->lru); 176 kref_get(&bo->list_kref); 177 178 if (bo->ttm != NULL) { 179 list_add_tail(&bo->swap, &bo->glob->swap_lru); 180 kref_get(&bo->list_kref); 181 } 182 } 183 } 184 EXPORT_SYMBOL(ttm_bo_add_to_lru); 185 186 int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 187 { 188 int put_count = 0; 189 190 if (!list_empty(&bo->swap)) { 191 list_del_init(&bo->swap); 192 ++put_count; 193 } 194 if (!list_empty(&bo->lru)) { 195 list_del_init(&bo->lru); 196 ++put_count; 197 } 198 199 /* 200 * TODO: Add a driver hook to delete from 201 * driver-specific LRU's here. 202 */ 203 204 return put_count; 205 } 206 207 static void ttm_bo_ref_bug(struct kref *list_kref) 208 { 209 BUG(); 210 } 211 212 void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, 213 bool never_free) 214 { 215 kref_sub(&bo->list_kref, count, 216 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); 217 } 218 219 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) 220 { 221 int put_count; 222 223 spin_lock(&bo->glob->lru_lock); 224 put_count = ttm_bo_del_from_lru(bo); 225 spin_unlock(&bo->glob->lru_lock); 226 ttm_bo_list_ref_sub(bo, put_count, true); 227 } 228 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); 229 230 /* 231 * Call bo->mutex locked. 232 */ 233 static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 234 { 235 struct ttm_bo_device *bdev = bo->bdev; 236 struct ttm_bo_global *glob = bo->glob; 237 int ret = 0; 238 uint32_t page_flags = 0; 239 240 TTM_ASSERT_LOCKED(&bo->mutex); 241 bo->ttm = NULL; 242 243 if (bdev->need_dma32) 244 page_flags |= TTM_PAGE_FLAG_DMA32; 245 246 switch (bo->type) { 247 case ttm_bo_type_device: 248 if (zero_alloc) 249 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 250 case ttm_bo_type_kernel: 251 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 252 page_flags, glob->dummy_read_page); 253 if (unlikely(bo->ttm == NULL)) 254 ret = -ENOMEM; 255 break; 256 case ttm_bo_type_sg: 257 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 258 page_flags | TTM_PAGE_FLAG_SG, 259 glob->dummy_read_page); 260 if (unlikely(bo->ttm == NULL)) { 261 ret = -ENOMEM; 262 break; 263 } 264 bo->ttm->sg = bo->sg; 265 break; 266 default: 267 pr_err("Illegal buffer object type\n"); 268 ret = -EINVAL; 269 break; 270 } 271 272 return ret; 273 } 274 275 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 276 struct ttm_mem_reg *mem, 277 bool evict, bool interruptible, 278 bool no_wait_gpu) 279 { 280 struct ttm_bo_device *bdev = bo->bdev; 281 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 282 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 283 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 284 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 285 int ret = 0; 286 287 if (old_is_pci || new_is_pci || 288 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 289 ret = ttm_mem_io_lock(old_man, true); 290 if (unlikely(ret != 0)) 291 goto out_err; 292 ttm_bo_unmap_virtual_locked(bo); 293 ttm_mem_io_unlock(old_man); 294 } 295 296 /* 297 * Create and bind a ttm if required. 298 */ 299 300 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 301 if (bo->ttm == NULL) { 302 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 303 ret = ttm_bo_add_ttm(bo, zero); 304 if (ret) 305 goto out_err; 306 } 307 308 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 309 if (ret) 310 goto out_err; 311 312 if (mem->mem_type != TTM_PL_SYSTEM) { 313 ret = ttm_tt_bind(bo->ttm, mem); 314 if (ret) 315 goto out_err; 316 } 317 318 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 319 if (bdev->driver->move_notify) 320 bdev->driver->move_notify(bo, mem); 321 bo->mem = *mem; 322 mem->mm_node = NULL; 323 goto moved; 324 } 325 } 326 327 if (bdev->driver->move_notify) 328 bdev->driver->move_notify(bo, mem); 329 330 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 331 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 332 ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); 333 else if (bdev->driver->move) 334 ret = bdev->driver->move(bo, evict, interruptible, 335 no_wait_gpu, mem); 336 else 337 ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); 338 339 if (ret) { 340 if (bdev->driver->move_notify) { 341 struct ttm_mem_reg tmp_mem = *mem; 342 *mem = bo->mem; 343 bo->mem = tmp_mem; 344 bdev->driver->move_notify(bo, mem); 345 bo->mem = *mem; 346 *mem = tmp_mem; 347 } 348 349 goto out_err; 350 } 351 352 moved: 353 if (bo->evicted) { 354 if (bdev->driver->invalidate_caches) { 355 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 356 if (ret) 357 pr_err("Can not flush read caches\n"); 358 } 359 bo->evicted = false; 360 } 361 362 if (bo->mem.mm_node) { 363 bo->offset = (bo->mem.start << PAGE_SHIFT) + 364 bdev->man[bo->mem.mem_type].gpu_offset; 365 bo->cur_placement = bo->mem.placement; 366 } else 367 bo->offset = 0; 368 369 return 0; 370 371 out_err: 372 new_man = &bdev->man[bo->mem.mem_type]; 373 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { 374 ttm_tt_unbind(bo->ttm); 375 ttm_tt_destroy(bo->ttm); 376 bo->ttm = NULL; 377 } 378 379 return ret; 380 } 381 382 /** 383 * Call bo::reserved. 384 * Will release GPU memory type usage on destruction. 385 * This is the place to put in driver specific hooks to release 386 * driver private resources. 387 * Will release the bo::reserved lock. 388 */ 389 390 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 391 { 392 if (bo->bdev->driver->move_notify) 393 bo->bdev->driver->move_notify(bo, NULL); 394 395 if (bo->ttm) { 396 ttm_tt_unbind(bo->ttm); 397 ttm_tt_destroy(bo->ttm); 398 bo->ttm = NULL; 399 } 400 ttm_bo_mem_put(bo, &bo->mem); 401 402 ww_mutex_unlock (&bo->resv->lock); 403 } 404 405 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 406 { 407 struct ttm_bo_device *bdev = bo->bdev; 408 struct ttm_bo_global *glob = bo->glob; 409 struct ttm_bo_driver *driver = bdev->driver; 410 void *sync_obj = NULL; 411 int put_count; 412 int ret; 413 414 spin_lock(&glob->lru_lock); 415 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 416 417 spin_lock(&bdev->fence_lock); 418 (void) ttm_bo_wait(bo, false, false, true); 419 if (!ret && !bo->sync_obj) { 420 spin_unlock(&bdev->fence_lock); 421 put_count = ttm_bo_del_from_lru(bo); 422 423 spin_unlock(&glob->lru_lock); 424 ttm_bo_cleanup_memtype_use(bo); 425 426 ttm_bo_list_ref_sub(bo, put_count, true); 427 428 return; 429 } 430 if (bo->sync_obj) 431 sync_obj = driver->sync_obj_ref(bo->sync_obj); 432 spin_unlock(&bdev->fence_lock); 433 434 if (!ret) { 435 436 /* 437 * Make NO_EVICT bos immediately available to 438 * shrinkers, now that they are queued for 439 * destruction. 440 */ 441 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { 442 bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; 443 ttm_bo_add_to_lru(bo); 444 } 445 446 ww_mutex_unlock(&bo->resv->lock); 447 } 448 449 kref_get(&bo->list_kref); 450 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 451 spin_unlock(&glob->lru_lock); 452 453 if (sync_obj) { 454 driver->sync_obj_flush(sync_obj); 455 driver->sync_obj_unref(&sync_obj); 456 } 457 schedule_delayed_work(&bdev->wq, 458 ((HZ / 100) < 1) ? 1 : HZ / 100); 459 } 460 461 /** 462 * function ttm_bo_cleanup_refs_and_unlock 463 * If bo idle, remove from delayed- and lru lists, and unref. 464 * If not idle, do nothing. 465 * 466 * Must be called with lru_lock and reservation held, this function 467 * will drop both before returning. 468 * 469 * @interruptible Any sleeps should occur interruptibly. 470 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 471 */ 472 473 static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, 474 bool interruptible, 475 bool no_wait_gpu) 476 { 477 struct ttm_bo_device *bdev = bo->bdev; 478 struct ttm_bo_driver *driver = bdev->driver; 479 struct ttm_bo_global *glob = bo->glob; 480 int put_count; 481 int ret; 482 483 spin_lock(&bdev->fence_lock); 484 ret = ttm_bo_wait(bo, false, false, true); 485 486 if (ret && !no_wait_gpu) { 487 void *sync_obj; 488 489 /* 490 * Take a reference to the fence and unreserve, 491 * at this point the buffer should be dead, so 492 * no new sync objects can be attached. 493 */ 494 sync_obj = driver->sync_obj_ref(bo->sync_obj); 495 spin_unlock(&bdev->fence_lock); 496 497 ww_mutex_unlock(&bo->resv->lock); 498 spin_unlock(&glob->lru_lock); 499 500 ret = driver->sync_obj_wait(sync_obj, false, interruptible); 501 driver->sync_obj_unref(&sync_obj); 502 if (ret) 503 return ret; 504 505 /* 506 * remove sync_obj with ttm_bo_wait, the wait should be 507 * finished, and no new wait object should have been added. 508 */ 509 spin_lock(&bdev->fence_lock); 510 ret = ttm_bo_wait(bo, false, false, true); 511 WARN_ON(ret); 512 spin_unlock(&bdev->fence_lock); 513 if (ret) 514 return ret; 515 516 spin_lock(&glob->lru_lock); 517 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 518 519 /* 520 * We raced, and lost, someone else holds the reservation now, 521 * and is probably busy in ttm_bo_cleanup_memtype_use. 522 * 523 * Even if it's not the case, because we finished waiting any 524 * delayed destruction would succeed, so just return success 525 * here. 526 */ 527 if (ret) { 528 spin_unlock(&glob->lru_lock); 529 return 0; 530 } 531 } else 532 spin_unlock(&bdev->fence_lock); 533 534 if (ret || unlikely(list_empty(&bo->ddestroy))) { 535 ww_mutex_unlock(&bo->resv->lock); 536 spin_unlock(&glob->lru_lock); 537 return ret; 538 } 539 540 put_count = ttm_bo_del_from_lru(bo); 541 list_del_init(&bo->ddestroy); 542 ++put_count; 543 544 spin_unlock(&glob->lru_lock); 545 ttm_bo_cleanup_memtype_use(bo); 546 547 ttm_bo_list_ref_sub(bo, put_count, true); 548 549 return 0; 550 } 551 552 /** 553 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 554 * encountered buffers. 555 */ 556 557 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 558 { 559 struct ttm_bo_global *glob = bdev->glob; 560 struct ttm_buffer_object *entry = NULL; 561 int ret = 0; 562 563 spin_lock(&glob->lru_lock); 564 if (list_empty(&bdev->ddestroy)) 565 goto out_unlock; 566 567 entry = list_first_entry(&bdev->ddestroy, 568 struct ttm_buffer_object, ddestroy); 569 kref_get(&entry->list_kref); 570 571 for (;;) { 572 struct ttm_buffer_object *nentry = NULL; 573 574 if (entry->ddestroy.next != &bdev->ddestroy) { 575 nentry = list_first_entry(&entry->ddestroy, 576 struct ttm_buffer_object, ddestroy); 577 kref_get(&nentry->list_kref); 578 } 579 580 ret = ttm_bo_reserve_nolru(entry, false, true, false, 0); 581 if (remove_all && ret) { 582 spin_unlock(&glob->lru_lock); 583 ret = ttm_bo_reserve_nolru(entry, false, false, 584 false, 0); 585 spin_lock(&glob->lru_lock); 586 } 587 588 if (!ret) 589 ret = ttm_bo_cleanup_refs_and_unlock(entry, false, 590 !remove_all); 591 else 592 spin_unlock(&glob->lru_lock); 593 594 kref_put(&entry->list_kref, ttm_bo_release_list); 595 entry = nentry; 596 597 if (ret || !entry) 598 goto out; 599 600 spin_lock(&glob->lru_lock); 601 if (list_empty(&entry->ddestroy)) 602 break; 603 } 604 605 out_unlock: 606 spin_unlock(&glob->lru_lock); 607 out: 608 if (entry) 609 kref_put(&entry->list_kref, ttm_bo_release_list); 610 return ret; 611 } 612 613 static void ttm_bo_delayed_workqueue(struct work_struct *work) 614 { 615 struct ttm_bo_device *bdev = 616 container_of(work, struct ttm_bo_device, wq.work); 617 618 if (ttm_bo_delayed_delete(bdev, false)) { 619 schedule_delayed_work(&bdev->wq, 620 ((HZ / 100) < 1) ? 1 : HZ / 100); 621 } 622 } 623 624 static void ttm_bo_release(struct kref *kref) 625 { 626 struct ttm_buffer_object *bo = 627 container_of(kref, struct ttm_buffer_object, kref); 628 struct ttm_bo_device *bdev = bo->bdev; 629 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 630 631 drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); 632 ttm_mem_io_lock(man, false); 633 ttm_mem_io_free_vm(bo); 634 ttm_mem_io_unlock(man); 635 ttm_bo_cleanup_refs_or_queue(bo); 636 kref_put(&bo->list_kref, ttm_bo_release_list); 637 } 638 639 void ttm_bo_unref(struct ttm_buffer_object **p_bo) 640 { 641 struct ttm_buffer_object *bo = *p_bo; 642 643 *p_bo = NULL; 644 kref_put(&bo->kref, ttm_bo_release); 645 } 646 EXPORT_SYMBOL(ttm_bo_unref); 647 648 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 649 { 650 return cancel_delayed_work_sync(&bdev->wq); 651 } 652 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 653 654 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 655 { 656 if (resched) 657 schedule_delayed_work(&bdev->wq, 658 ((HZ / 100) < 1) ? 1 : HZ / 100); 659 } 660 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 661 662 static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 663 bool no_wait_gpu) 664 { 665 struct ttm_bo_device *bdev = bo->bdev; 666 struct ttm_mem_reg evict_mem; 667 struct ttm_placement placement; 668 int ret = 0; 669 670 spin_lock(&bdev->fence_lock); 671 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 672 spin_unlock(&bdev->fence_lock); 673 674 if (unlikely(ret != 0)) { 675 if (ret != -ERESTARTSYS) { 676 pr_err("Failed to expire sync object before buffer eviction\n"); 677 } 678 goto out; 679 } 680 681 lockdep_assert_held(&bo->resv->lock.base); 682 683 evict_mem = bo->mem; 684 evict_mem.mm_node = NULL; 685 evict_mem.bus.io_reserved_vm = false; 686 evict_mem.bus.io_reserved_count = 0; 687 688 placement.fpfn = 0; 689 placement.lpfn = 0; 690 placement.num_placement = 0; 691 placement.num_busy_placement = 0; 692 bdev->driver->evict_flags(bo, &placement); 693 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 694 no_wait_gpu); 695 if (ret) { 696 if (ret != -ERESTARTSYS) { 697 pr_err("Failed to find memory space for buffer 0x%p eviction\n", 698 bo); 699 ttm_bo_mem_space_debug(bo, &placement); 700 } 701 goto out; 702 } 703 704 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 705 no_wait_gpu); 706 if (ret) { 707 if (ret != -ERESTARTSYS) 708 pr_err("Buffer eviction failed\n"); 709 ttm_bo_mem_put(bo, &evict_mem); 710 goto out; 711 } 712 bo->evicted = true; 713 out: 714 return ret; 715 } 716 717 static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 718 uint32_t mem_type, 719 bool interruptible, 720 bool no_wait_gpu) 721 { 722 struct ttm_bo_global *glob = bdev->glob; 723 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 724 struct ttm_buffer_object *bo; 725 int ret = -EBUSY, put_count; 726 727 spin_lock(&glob->lru_lock); 728 list_for_each_entry(bo, &man->lru, lru) { 729 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 730 if (!ret) 731 break; 732 } 733 734 if (ret) { 735 spin_unlock(&glob->lru_lock); 736 return ret; 737 } 738 739 kref_get(&bo->list_kref); 740 741 if (!list_empty(&bo->ddestroy)) { 742 ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, 743 no_wait_gpu); 744 kref_put(&bo->list_kref, ttm_bo_release_list); 745 return ret; 746 } 747 748 put_count = ttm_bo_del_from_lru(bo); 749 spin_unlock(&glob->lru_lock); 750 751 BUG_ON(ret != 0); 752 753 ttm_bo_list_ref_sub(bo, put_count, true); 754 755 ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); 756 ttm_bo_unreserve(bo); 757 758 kref_put(&bo->list_kref, ttm_bo_release_list); 759 return ret; 760 } 761 762 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 763 { 764 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 765 766 if (mem->mm_node) 767 (*man->func->put_node)(man, mem); 768 } 769 EXPORT_SYMBOL(ttm_bo_mem_put); 770 771 /** 772 * Repeatedly evict memory from the LRU for @mem_type until we create enough 773 * space, or we've evicted everything and there isn't enough space. 774 */ 775 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 776 uint32_t mem_type, 777 struct ttm_placement *placement, 778 struct ttm_mem_reg *mem, 779 bool interruptible, 780 bool no_wait_gpu) 781 { 782 struct ttm_bo_device *bdev = bo->bdev; 783 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 784 int ret; 785 786 do { 787 ret = (*man->func->get_node)(man, bo, placement, mem); 788 if (unlikely(ret != 0)) 789 return ret; 790 if (mem->mm_node) 791 break; 792 ret = ttm_mem_evict_first(bdev, mem_type, 793 interruptible, no_wait_gpu); 794 if (unlikely(ret != 0)) 795 return ret; 796 } while (1); 797 if (mem->mm_node == NULL) 798 return -ENOMEM; 799 mem->mem_type = mem_type; 800 return 0; 801 } 802 803 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 804 uint32_t cur_placement, 805 uint32_t proposed_placement) 806 { 807 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 808 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 809 810 /** 811 * Keep current caching if possible. 812 */ 813 814 if ((cur_placement & caching) != 0) 815 result |= (cur_placement & caching); 816 else if ((man->default_caching & caching) != 0) 817 result |= man->default_caching; 818 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 819 result |= TTM_PL_FLAG_CACHED; 820 else if ((TTM_PL_FLAG_WC & caching) != 0) 821 result |= TTM_PL_FLAG_WC; 822 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 823 result |= TTM_PL_FLAG_UNCACHED; 824 825 return result; 826 } 827 828 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 829 uint32_t mem_type, 830 uint32_t proposed_placement, 831 uint32_t *masked_placement) 832 { 833 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 834 835 if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) 836 return false; 837 838 if ((proposed_placement & man->available_caching) == 0) 839 return false; 840 841 cur_flags |= (proposed_placement & man->available_caching); 842 843 *masked_placement = cur_flags; 844 return true; 845 } 846 847 /** 848 * Creates space for memory region @mem according to its type. 849 * 850 * This function first searches for free space in compatible memory types in 851 * the priority order defined by the driver. If free space isn't found, then 852 * ttm_bo_mem_force_space is attempted in priority order to evict and find 853 * space. 854 */ 855 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 856 struct ttm_placement *placement, 857 struct ttm_mem_reg *mem, 858 bool interruptible, 859 bool no_wait_gpu) 860 { 861 struct ttm_bo_device *bdev = bo->bdev; 862 struct ttm_mem_type_manager *man; 863 uint32_t mem_type = TTM_PL_SYSTEM; 864 uint32_t cur_flags = 0; 865 bool type_found = false; 866 bool type_ok = false; 867 bool has_erestartsys = false; 868 int i, ret; 869 870 mem->mm_node = NULL; 871 for (i = 0; i < placement->num_placement; ++i) { 872 ret = ttm_mem_type_from_flags(placement->placement[i], 873 &mem_type); 874 if (ret) 875 return ret; 876 man = &bdev->man[mem_type]; 877 878 type_ok = ttm_bo_mt_compatible(man, 879 mem_type, 880 placement->placement[i], 881 &cur_flags); 882 883 if (!type_ok) 884 continue; 885 886 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 887 cur_flags); 888 /* 889 * Use the access and other non-mapping-related flag bits from 890 * the memory placement flags to the current flags 891 */ 892 ttm_flag_masked(&cur_flags, placement->placement[i], 893 ~TTM_PL_MASK_MEMTYPE); 894 895 if (mem_type == TTM_PL_SYSTEM) 896 break; 897 898 if (man->has_type && man->use_type) { 899 type_found = true; 900 ret = (*man->func->get_node)(man, bo, placement, mem); 901 if (unlikely(ret)) 902 return ret; 903 } 904 if (mem->mm_node) 905 break; 906 } 907 908 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 909 mem->mem_type = mem_type; 910 mem->placement = cur_flags; 911 return 0; 912 } 913 914 if (!type_found) 915 return -EINVAL; 916 917 for (i = 0; i < placement->num_busy_placement; ++i) { 918 ret = ttm_mem_type_from_flags(placement->busy_placement[i], 919 &mem_type); 920 if (ret) 921 return ret; 922 man = &bdev->man[mem_type]; 923 if (!man->has_type) 924 continue; 925 if (!ttm_bo_mt_compatible(man, 926 mem_type, 927 placement->busy_placement[i], 928 &cur_flags)) 929 continue; 930 931 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 932 cur_flags); 933 /* 934 * Use the access and other non-mapping-related flag bits from 935 * the memory placement flags to the current flags 936 */ 937 ttm_flag_masked(&cur_flags, placement->busy_placement[i], 938 ~TTM_PL_MASK_MEMTYPE); 939 940 941 if (mem_type == TTM_PL_SYSTEM) { 942 mem->mem_type = mem_type; 943 mem->placement = cur_flags; 944 mem->mm_node = NULL; 945 return 0; 946 } 947 948 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, 949 interruptible, no_wait_gpu); 950 if (ret == 0 && mem->mm_node) { 951 mem->placement = cur_flags; 952 return 0; 953 } 954 if (ret == -ERESTARTSYS) 955 has_erestartsys = true; 956 } 957 ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; 958 return ret; 959 } 960 EXPORT_SYMBOL(ttm_bo_mem_space); 961 962 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 963 struct ttm_placement *placement, 964 bool interruptible, 965 bool no_wait_gpu) 966 { 967 int ret = 0; 968 struct ttm_mem_reg mem; 969 struct ttm_bo_device *bdev = bo->bdev; 970 971 lockdep_assert_held(&bo->resv->lock.base); 972 973 /* 974 * FIXME: It's possible to pipeline buffer moves. 975 * Have the driver move function wait for idle when necessary, 976 * instead of doing it here. 977 */ 978 spin_lock(&bdev->fence_lock); 979 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 980 spin_unlock(&bdev->fence_lock); 981 if (ret) 982 return ret; 983 mem.num_pages = bo->num_pages; 984 mem.size = mem.num_pages << PAGE_SHIFT; 985 mem.page_alignment = bo->mem.page_alignment; 986 mem.bus.io_reserved_vm = false; 987 mem.bus.io_reserved_count = 0; 988 /* 989 * Determine where to move the buffer. 990 */ 991 ret = ttm_bo_mem_space(bo, placement, &mem, 992 interruptible, no_wait_gpu); 993 if (ret) 994 goto out_unlock; 995 ret = ttm_bo_handle_move_mem(bo, &mem, false, 996 interruptible, no_wait_gpu); 997 out_unlock: 998 if (ret && mem.mm_node) 999 ttm_bo_mem_put(bo, &mem); 1000 return ret; 1001 } 1002 1003 static bool ttm_bo_mem_compat(struct ttm_placement *placement, 1004 struct ttm_mem_reg *mem, 1005 uint32_t *new_flags) 1006 { 1007 int i; 1008 1009 if (mem->mm_node && placement->lpfn != 0 && 1010 (mem->start < placement->fpfn || 1011 mem->start + mem->num_pages > placement->lpfn)) 1012 return false; 1013 1014 for (i = 0; i < placement->num_placement; i++) { 1015 *new_flags = placement->placement[i]; 1016 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1017 (*new_flags & mem->placement & TTM_PL_MASK_MEM)) 1018 return true; 1019 } 1020 1021 for (i = 0; i < placement->num_busy_placement; i++) { 1022 *new_flags = placement->busy_placement[i]; 1023 if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && 1024 (*new_flags & mem->placement & TTM_PL_MASK_MEM)) 1025 return true; 1026 } 1027 1028 return false; 1029 } 1030 1031 int ttm_bo_validate(struct ttm_buffer_object *bo, 1032 struct ttm_placement *placement, 1033 bool interruptible, 1034 bool no_wait_gpu) 1035 { 1036 int ret; 1037 uint32_t new_flags; 1038 1039 lockdep_assert_held(&bo->resv->lock.base); 1040 /* Check that range is valid */ 1041 if (placement->lpfn || placement->fpfn) 1042 if (placement->fpfn > placement->lpfn || 1043 (placement->lpfn - placement->fpfn) < bo->num_pages) 1044 return -EINVAL; 1045 /* 1046 * Check whether we need to move buffer. 1047 */ 1048 if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { 1049 ret = ttm_bo_move_buffer(bo, placement, interruptible, 1050 no_wait_gpu); 1051 if (ret) 1052 return ret; 1053 } else { 1054 /* 1055 * Use the access and other non-mapping-related flag bits from 1056 * the compatible memory placement flags to the active flags 1057 */ 1058 ttm_flag_masked(&bo->mem.placement, new_flags, 1059 ~TTM_PL_MASK_MEMTYPE); 1060 } 1061 /* 1062 * We might need to add a TTM. 1063 */ 1064 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1065 ret = ttm_bo_add_ttm(bo, true); 1066 if (ret) 1067 return ret; 1068 } 1069 return 0; 1070 } 1071 EXPORT_SYMBOL(ttm_bo_validate); 1072 1073 int ttm_bo_check_placement(struct ttm_buffer_object *bo, 1074 struct ttm_placement *placement) 1075 { 1076 BUG_ON((placement->fpfn || placement->lpfn) && 1077 (bo->mem.num_pages > (placement->lpfn - placement->fpfn))); 1078 1079 return 0; 1080 } 1081 1082 int ttm_bo_init(struct ttm_bo_device *bdev, 1083 struct ttm_buffer_object *bo, 1084 unsigned long size, 1085 enum ttm_bo_type type, 1086 struct ttm_placement *placement, 1087 uint32_t page_alignment, 1088 bool interruptible, 1089 struct file *persistent_swap_storage, 1090 size_t acc_size, 1091 struct sg_table *sg, 1092 void (*destroy) (struct ttm_buffer_object *)) 1093 { 1094 int ret = 0; 1095 unsigned long num_pages; 1096 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1097 bool locked; 1098 1099 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1100 if (ret) { 1101 pr_err("Out of kernel memory\n"); 1102 if (destroy) 1103 (*destroy)(bo); 1104 else 1105 kfree(bo); 1106 return -ENOMEM; 1107 } 1108 1109 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1110 if (num_pages == 0) { 1111 pr_err("Illegal buffer object size\n"); 1112 if (destroy) 1113 (*destroy)(bo); 1114 else 1115 kfree(bo); 1116 ttm_mem_global_free(mem_glob, acc_size); 1117 return -EINVAL; 1118 } 1119 bo->destroy = destroy; 1120 1121 kref_init(&bo->kref); 1122 kref_init(&bo->list_kref); 1123 atomic_set(&bo->cpu_writers, 0); 1124 INIT_LIST_HEAD(&bo->lru); 1125 INIT_LIST_HEAD(&bo->ddestroy); 1126 INIT_LIST_HEAD(&bo->swap); 1127 INIT_LIST_HEAD(&bo->io_reserve_lru); 1128 mutex_init(&bo->wu_mutex); 1129 bo->bdev = bdev; 1130 bo->glob = bdev->glob; 1131 bo->type = type; 1132 bo->num_pages = num_pages; 1133 bo->mem.size = num_pages << PAGE_SHIFT; 1134 bo->mem.mem_type = TTM_PL_SYSTEM; 1135 bo->mem.num_pages = bo->num_pages; 1136 bo->mem.mm_node = NULL; 1137 bo->mem.page_alignment = page_alignment; 1138 bo->mem.bus.io_reserved_vm = false; 1139 bo->mem.bus.io_reserved_count = 0; 1140 bo->priv_flags = 0; 1141 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1142 bo->persistent_swap_storage = persistent_swap_storage; 1143 bo->acc_size = acc_size; 1144 bo->sg = sg; 1145 bo->resv = &bo->ttm_resv; 1146 reservation_object_init(bo->resv); 1147 atomic_inc(&bo->glob->bo_count); 1148 drm_vma_node_reset(&bo->vma_node); 1149 1150 ret = ttm_bo_check_placement(bo, placement); 1151 1152 /* 1153 * For ttm_bo_type_device buffers, allocate 1154 * address space from the device. 1155 */ 1156 if (likely(!ret) && 1157 (bo->type == ttm_bo_type_device || 1158 bo->type == ttm_bo_type_sg)) 1159 ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, 1160 bo->mem.num_pages); 1161 1162 locked = ww_mutex_trylock(&bo->resv->lock); 1163 WARN_ON(!locked); 1164 1165 if (likely(!ret)) 1166 ret = ttm_bo_validate(bo, placement, interruptible, false); 1167 1168 ttm_bo_unreserve(bo); 1169 1170 if (unlikely(ret)) 1171 ttm_bo_unref(&bo); 1172 1173 return ret; 1174 } 1175 EXPORT_SYMBOL(ttm_bo_init); 1176 1177 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1178 unsigned long bo_size, 1179 unsigned struct_size) 1180 { 1181 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1182 size_t size = 0; 1183 1184 size += ttm_round_pot(struct_size); 1185 size += PAGE_ALIGN(npages * sizeof(void *)); 1186 size += ttm_round_pot(sizeof(struct ttm_tt)); 1187 return size; 1188 } 1189 EXPORT_SYMBOL(ttm_bo_acc_size); 1190 1191 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1192 unsigned long bo_size, 1193 unsigned struct_size) 1194 { 1195 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1196 size_t size = 0; 1197 1198 size += ttm_round_pot(struct_size); 1199 size += PAGE_ALIGN(npages * sizeof(void *)); 1200 size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); 1201 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1202 return size; 1203 } 1204 EXPORT_SYMBOL(ttm_bo_dma_acc_size); 1205 1206 int ttm_bo_create(struct ttm_bo_device *bdev, 1207 unsigned long size, 1208 enum ttm_bo_type type, 1209 struct ttm_placement *placement, 1210 uint32_t page_alignment, 1211 bool interruptible, 1212 struct file *persistent_swap_storage, 1213 struct ttm_buffer_object **p_bo) 1214 { 1215 struct ttm_buffer_object *bo; 1216 size_t acc_size; 1217 int ret; 1218 1219 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1220 if (unlikely(bo == NULL)) 1221 return -ENOMEM; 1222 1223 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1224 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1225 interruptible, persistent_swap_storage, acc_size, 1226 NULL, NULL); 1227 if (likely(ret == 0)) 1228 *p_bo = bo; 1229 1230 return ret; 1231 } 1232 EXPORT_SYMBOL(ttm_bo_create); 1233 1234 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1235 unsigned mem_type, bool allow_errors) 1236 { 1237 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1238 struct ttm_bo_global *glob = bdev->glob; 1239 int ret; 1240 1241 /* 1242 * Can't use standard list traversal since we're unlocking. 1243 */ 1244 1245 spin_lock(&glob->lru_lock); 1246 while (!list_empty(&man->lru)) { 1247 spin_unlock(&glob->lru_lock); 1248 ret = ttm_mem_evict_first(bdev, mem_type, false, false); 1249 if (ret) { 1250 if (allow_errors) { 1251 return ret; 1252 } else { 1253 pr_err("Cleanup eviction failed\n"); 1254 } 1255 } 1256 spin_lock(&glob->lru_lock); 1257 } 1258 spin_unlock(&glob->lru_lock); 1259 return 0; 1260 } 1261 1262 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1263 { 1264 struct ttm_mem_type_manager *man; 1265 int ret = -EINVAL; 1266 1267 if (mem_type >= TTM_NUM_MEM_TYPES) { 1268 pr_err("Illegal memory type %d\n", mem_type); 1269 return ret; 1270 } 1271 man = &bdev->man[mem_type]; 1272 1273 if (!man->has_type) { 1274 pr_err("Trying to take down uninitialized memory manager type %u\n", 1275 mem_type); 1276 return ret; 1277 } 1278 1279 man->use_type = false; 1280 man->has_type = false; 1281 1282 ret = 0; 1283 if (mem_type > 0) { 1284 ttm_bo_force_list_clean(bdev, mem_type, false); 1285 1286 ret = (*man->func->takedown)(man); 1287 } 1288 1289 return ret; 1290 } 1291 EXPORT_SYMBOL(ttm_bo_clean_mm); 1292 1293 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1294 { 1295 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1296 1297 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1298 pr_err("Illegal memory manager memory type %u\n", mem_type); 1299 return -EINVAL; 1300 } 1301 1302 if (!man->has_type) { 1303 pr_err("Memory type %u has not been initialized\n", mem_type); 1304 return 0; 1305 } 1306 1307 return ttm_bo_force_list_clean(bdev, mem_type, true); 1308 } 1309 EXPORT_SYMBOL(ttm_bo_evict_mm); 1310 1311 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1312 unsigned long p_size) 1313 { 1314 int ret = -EINVAL; 1315 struct ttm_mem_type_manager *man; 1316 1317 BUG_ON(type >= TTM_NUM_MEM_TYPES); 1318 man = &bdev->man[type]; 1319 BUG_ON(man->has_type); 1320 man->io_reserve_fastpath = true; 1321 man->use_io_reserve_lru = false; 1322 mutex_init(&man->io_reserve_mutex); 1323 INIT_LIST_HEAD(&man->io_reserve_lru); 1324 1325 ret = bdev->driver->init_mem_type(bdev, type, man); 1326 if (ret) 1327 return ret; 1328 man->bdev = bdev; 1329 1330 ret = 0; 1331 if (type != TTM_PL_SYSTEM) { 1332 ret = (*man->func->init)(man, p_size); 1333 if (ret) 1334 return ret; 1335 } 1336 man->has_type = true; 1337 man->use_type = true; 1338 man->size = p_size; 1339 1340 INIT_LIST_HEAD(&man->lru); 1341 1342 return 0; 1343 } 1344 EXPORT_SYMBOL(ttm_bo_init_mm); 1345 1346 static void ttm_bo_global_kobj_release(struct kobject *kobj) 1347 { 1348 struct ttm_bo_global *glob = 1349 container_of(kobj, struct ttm_bo_global, kobj); 1350 1351 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1352 __free_page(glob->dummy_read_page); 1353 kfree(glob); 1354 } 1355 1356 void ttm_bo_global_release(struct drm_global_reference *ref) 1357 { 1358 struct ttm_bo_global *glob = ref->object; 1359 1360 kobject_del(&glob->kobj); 1361 kobject_put(&glob->kobj); 1362 } 1363 EXPORT_SYMBOL(ttm_bo_global_release); 1364 1365 int ttm_bo_global_init(struct drm_global_reference *ref) 1366 { 1367 struct ttm_bo_global_ref *bo_ref = 1368 container_of(ref, struct ttm_bo_global_ref, ref); 1369 struct ttm_bo_global *glob = ref->object; 1370 int ret; 1371 1372 mutex_init(&glob->device_list_mutex); 1373 spin_lock_init(&glob->lru_lock); 1374 glob->mem_glob = bo_ref->mem_glob; 1375 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); 1376 1377 if (unlikely(glob->dummy_read_page == NULL)) { 1378 ret = -ENOMEM; 1379 goto out_no_drp; 1380 } 1381 1382 INIT_LIST_HEAD(&glob->swap_lru); 1383 INIT_LIST_HEAD(&glob->device_list); 1384 1385 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1386 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1387 if (unlikely(ret != 0)) { 1388 pr_err("Could not register buffer object swapout\n"); 1389 goto out_no_shrink; 1390 } 1391 1392 atomic_set(&glob->bo_count, 0); 1393 1394 ret = kobject_init_and_add( 1395 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); 1396 if (unlikely(ret != 0)) 1397 kobject_put(&glob->kobj); 1398 return ret; 1399 out_no_shrink: 1400 __free_page(glob->dummy_read_page); 1401 out_no_drp: 1402 kfree(glob); 1403 return ret; 1404 } 1405 EXPORT_SYMBOL(ttm_bo_global_init); 1406 1407 1408 int ttm_bo_device_release(struct ttm_bo_device *bdev) 1409 { 1410 int ret = 0; 1411 unsigned i = TTM_NUM_MEM_TYPES; 1412 struct ttm_mem_type_manager *man; 1413 struct ttm_bo_global *glob = bdev->glob; 1414 1415 while (i--) { 1416 man = &bdev->man[i]; 1417 if (man->has_type) { 1418 man->use_type = false; 1419 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1420 ret = -EBUSY; 1421 pr_err("DRM memory manager type %d is not clean\n", 1422 i); 1423 } 1424 man->has_type = false; 1425 } 1426 } 1427 1428 mutex_lock(&glob->device_list_mutex); 1429 list_del(&bdev->device_list); 1430 mutex_unlock(&glob->device_list_mutex); 1431 1432 cancel_delayed_work_sync(&bdev->wq); 1433 1434 while (ttm_bo_delayed_delete(bdev, true)) 1435 ; 1436 1437 spin_lock(&glob->lru_lock); 1438 if (list_empty(&bdev->ddestroy)) 1439 TTM_DEBUG("Delayed destroy list was clean\n"); 1440 1441 if (list_empty(&bdev->man[0].lru)) 1442 TTM_DEBUG("Swap list was clean\n"); 1443 spin_unlock(&glob->lru_lock); 1444 1445 drm_vma_offset_manager_destroy(&bdev->vma_manager); 1446 1447 return ret; 1448 } 1449 EXPORT_SYMBOL(ttm_bo_device_release); 1450 1451 int ttm_bo_device_init(struct ttm_bo_device *bdev, 1452 struct ttm_bo_global *glob, 1453 struct ttm_bo_driver *driver, 1454 uint64_t file_page_offset, 1455 bool need_dma32) 1456 { 1457 int ret = -EINVAL; 1458 1459 bdev->driver = driver; 1460 1461 memset(bdev->man, 0, sizeof(bdev->man)); 1462 1463 /* 1464 * Initialize the system memory buffer type. 1465 * Other types need to be driver / IOCTL initialized. 1466 */ 1467 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1468 if (unlikely(ret != 0)) 1469 goto out_no_sys; 1470 1471 drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, 1472 0x10000000); 1473 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); 1474 INIT_LIST_HEAD(&bdev->ddestroy); 1475 bdev->dev_mapping = NULL; 1476 bdev->glob = glob; 1477 bdev->need_dma32 = need_dma32; 1478 bdev->val_seq = 0; 1479 spin_lock_init(&bdev->fence_lock); 1480 mutex_lock(&glob->device_list_mutex); 1481 list_add_tail(&bdev->device_list, &glob->device_list); 1482 mutex_unlock(&glob->device_list_mutex); 1483 1484 return 0; 1485 out_no_sys: 1486 return ret; 1487 } 1488 EXPORT_SYMBOL(ttm_bo_device_init); 1489 1490 /* 1491 * buffer object vm functions. 1492 */ 1493 1494 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1495 { 1496 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1497 1498 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1499 if (mem->mem_type == TTM_PL_SYSTEM) 1500 return false; 1501 1502 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1503 return false; 1504 1505 if (mem->placement & TTM_PL_FLAG_CACHED) 1506 return false; 1507 } 1508 return true; 1509 } 1510 1511 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1512 { 1513 struct ttm_bo_device *bdev = bo->bdev; 1514 1515 drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping); 1516 ttm_mem_io_free_vm(bo); 1517 } 1518 1519 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1520 { 1521 struct ttm_bo_device *bdev = bo->bdev; 1522 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1523 1524 ttm_mem_io_lock(man, false); 1525 ttm_bo_unmap_virtual_locked(bo); 1526 ttm_mem_io_unlock(man); 1527 } 1528 1529 1530 EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1531 1532 1533 int ttm_bo_wait(struct ttm_buffer_object *bo, 1534 bool lazy, bool interruptible, bool no_wait) 1535 { 1536 struct ttm_bo_driver *driver = bo->bdev->driver; 1537 struct ttm_bo_device *bdev = bo->bdev; 1538 void *sync_obj; 1539 int ret = 0; 1540 1541 if (likely(bo->sync_obj == NULL)) 1542 return 0; 1543 1544 while (bo->sync_obj) { 1545 1546 if (driver->sync_obj_signaled(bo->sync_obj)) { 1547 void *tmp_obj = bo->sync_obj; 1548 bo->sync_obj = NULL; 1549 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1550 spin_unlock(&bdev->fence_lock); 1551 driver->sync_obj_unref(&tmp_obj); 1552 spin_lock(&bdev->fence_lock); 1553 continue; 1554 } 1555 1556 if (no_wait) 1557 return -EBUSY; 1558 1559 sync_obj = driver->sync_obj_ref(bo->sync_obj); 1560 spin_unlock(&bdev->fence_lock); 1561 ret = driver->sync_obj_wait(sync_obj, 1562 lazy, interruptible); 1563 if (unlikely(ret != 0)) { 1564 driver->sync_obj_unref(&sync_obj); 1565 spin_lock(&bdev->fence_lock); 1566 return ret; 1567 } 1568 spin_lock(&bdev->fence_lock); 1569 if (likely(bo->sync_obj == sync_obj)) { 1570 void *tmp_obj = bo->sync_obj; 1571 bo->sync_obj = NULL; 1572 clear_bit(TTM_BO_PRIV_FLAG_MOVING, 1573 &bo->priv_flags); 1574 spin_unlock(&bdev->fence_lock); 1575 driver->sync_obj_unref(&sync_obj); 1576 driver->sync_obj_unref(&tmp_obj); 1577 spin_lock(&bdev->fence_lock); 1578 } else { 1579 spin_unlock(&bdev->fence_lock); 1580 driver->sync_obj_unref(&sync_obj); 1581 spin_lock(&bdev->fence_lock); 1582 } 1583 } 1584 return 0; 1585 } 1586 EXPORT_SYMBOL(ttm_bo_wait); 1587 1588 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1589 { 1590 struct ttm_bo_device *bdev = bo->bdev; 1591 int ret = 0; 1592 1593 /* 1594 * Using ttm_bo_reserve makes sure the lru lists are updated. 1595 */ 1596 1597 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1598 if (unlikely(ret != 0)) 1599 return ret; 1600 spin_lock(&bdev->fence_lock); 1601 ret = ttm_bo_wait(bo, false, true, no_wait); 1602 spin_unlock(&bdev->fence_lock); 1603 if (likely(ret == 0)) 1604 atomic_inc(&bo->cpu_writers); 1605 ttm_bo_unreserve(bo); 1606 return ret; 1607 } 1608 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); 1609 1610 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1611 { 1612 atomic_dec(&bo->cpu_writers); 1613 } 1614 EXPORT_SYMBOL(ttm_bo_synccpu_write_release); 1615 1616 /** 1617 * A buffer object shrink method that tries to swap out the first 1618 * buffer object on the bo_global::swap_lru list. 1619 */ 1620 1621 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1622 { 1623 struct ttm_bo_global *glob = 1624 container_of(shrink, struct ttm_bo_global, shrink); 1625 struct ttm_buffer_object *bo; 1626 int ret = -EBUSY; 1627 int put_count; 1628 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); 1629 1630 spin_lock(&glob->lru_lock); 1631 list_for_each_entry(bo, &glob->swap_lru, swap) { 1632 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 1633 if (!ret) 1634 break; 1635 } 1636 1637 if (ret) { 1638 spin_unlock(&glob->lru_lock); 1639 return ret; 1640 } 1641 1642 kref_get(&bo->list_kref); 1643 1644 if (!list_empty(&bo->ddestroy)) { 1645 ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); 1646 kref_put(&bo->list_kref, ttm_bo_release_list); 1647 return ret; 1648 } 1649 1650 put_count = ttm_bo_del_from_lru(bo); 1651 spin_unlock(&glob->lru_lock); 1652 1653 ttm_bo_list_ref_sub(bo, put_count, true); 1654 1655 /** 1656 * Wait for GPU, then move to system cached. 1657 */ 1658 1659 spin_lock(&bo->bdev->fence_lock); 1660 ret = ttm_bo_wait(bo, false, false, false); 1661 spin_unlock(&bo->bdev->fence_lock); 1662 1663 if (unlikely(ret != 0)) 1664 goto out; 1665 1666 if ((bo->mem.placement & swap_placement) != swap_placement) { 1667 struct ttm_mem_reg evict_mem; 1668 1669 evict_mem = bo->mem; 1670 evict_mem.mm_node = NULL; 1671 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1672 evict_mem.mem_type = TTM_PL_SYSTEM; 1673 1674 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1675 false, false); 1676 if (unlikely(ret != 0)) 1677 goto out; 1678 } 1679 1680 ttm_bo_unmap_virtual(bo); 1681 1682 /** 1683 * Swap out. Buffer will be swapped in again as soon as 1684 * anyone tries to access a ttm page. 1685 */ 1686 1687 if (bo->bdev->driver->swap_notify) 1688 bo->bdev->driver->swap_notify(bo); 1689 1690 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1691 out: 1692 1693 /** 1694 * 1695 * Unreserve without putting on LRU to avoid swapping out an 1696 * already swapped buffer. 1697 */ 1698 1699 ww_mutex_unlock(&bo->resv->lock); 1700 kref_put(&bo->list_kref, ttm_bo_release_list); 1701 return ret; 1702 } 1703 1704 void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1705 { 1706 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1707 ; 1708 } 1709 EXPORT_SYMBOL(ttm_bo_swapout_all); 1710 1711 /** 1712 * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become 1713 * unreserved 1714 * 1715 * @bo: Pointer to buffer 1716 */ 1717 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) 1718 { 1719 int ret; 1720 1721 /* 1722 * In the absense of a wait_unlocked API, 1723 * Use the bo::wu_mutex to avoid triggering livelocks due to 1724 * concurrent use of this function. Note that this use of 1725 * bo::wu_mutex can go away if we change locking order to 1726 * mmap_sem -> bo::reserve. 1727 */ 1728 ret = mutex_lock_interruptible(&bo->wu_mutex); 1729 if (unlikely(ret != 0)) 1730 return -ERESTARTSYS; 1731 if (!ww_mutex_is_locked(&bo->resv->lock)) 1732 goto out_unlock; 1733 ret = ttm_bo_reserve_nolru(bo, true, false, false, NULL); 1734 if (unlikely(ret != 0)) 1735 goto out_unlock; 1736 ww_mutex_unlock(&bo->resv->lock); 1737 1738 out_unlock: 1739 mutex_unlock(&bo->wu_mutex); 1740 return ret; 1741 } 1742