1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2 /************************************************************************** 3 * 4 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 25 * USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 **************************************************************************/ 28 /* 29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 30 */ 31 32 #include <drm/ttm/ttm_bo_driver.h> 33 #include <drm/ttm/ttm_placement.h> 34 #include <drm/drm_vma_manager.h> 35 #include <linux/io.h> 36 #include <linux/highmem.h> 37 #include <linux/wait.h> 38 #include <linux/slab.h> 39 #include <linux/vmalloc.h> 40 #include <linux/module.h> 41 #include <linux/dma-resv.h> 42 43 struct ttm_transfer_obj { 44 struct ttm_buffer_object base; 45 struct ttm_buffer_object *bo; 46 }; 47 48 void ttm_bo_free_old_node(struct ttm_buffer_object *bo) 49 { 50 ttm_bo_mem_put(bo, &bo->mem); 51 } 52 53 int ttm_bo_move_ttm(struct ttm_buffer_object *bo, 54 struct ttm_operation_ctx *ctx, 55 struct ttm_mem_reg *new_mem) 56 { 57 struct ttm_tt *ttm = bo->ttm; 58 struct ttm_mem_reg *old_mem = &bo->mem; 59 int ret; 60 61 if (old_mem->mem_type != TTM_PL_SYSTEM) { 62 ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); 63 64 if (unlikely(ret != 0)) { 65 if (ret != -ERESTARTSYS) 66 pr_err("Failed to expire sync object before unbinding TTM\n"); 67 return ret; 68 } 69 70 ttm_tt_unbind(ttm); 71 ttm_bo_free_old_node(bo); 72 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, 73 TTM_PL_MASK_MEM); 74 old_mem->mem_type = TTM_PL_SYSTEM; 75 } 76 77 ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); 78 if (unlikely(ret != 0)) 79 return ret; 80 81 if (new_mem->mem_type != TTM_PL_SYSTEM) { 82 ret = ttm_tt_bind(ttm, new_mem, ctx); 83 if (unlikely(ret != 0)) 84 return ret; 85 } 86 87 *old_mem = *new_mem; 88 new_mem->mm_node = NULL; 89 90 return 0; 91 } 92 EXPORT_SYMBOL(ttm_bo_move_ttm); 93 94 int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) 95 { 96 if (likely(!man->use_io_reserve_lru)) 97 return 0; 98 99 if (interruptible) 100 return mutex_lock_interruptible(&man->io_reserve_mutex); 101 102 mutex_lock(&man->io_reserve_mutex); 103 return 0; 104 } 105 106 void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) 107 { 108 if (likely(!man->use_io_reserve_lru)) 109 return; 110 111 mutex_unlock(&man->io_reserve_mutex); 112 } 113 114 static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) 115 { 116 struct ttm_buffer_object *bo; 117 118 bo = list_first_entry_or_null(&man->io_reserve_lru, 119 struct ttm_buffer_object, 120 io_reserve_lru); 121 if (!bo) 122 return -ENOSPC; 123 124 list_del_init(&bo->io_reserve_lru); 125 ttm_bo_unmap_virtual_locked(bo); 126 return 0; 127 } 128 129 int ttm_mem_io_reserve(struct ttm_bo_device *bdev, 130 struct ttm_mem_reg *mem) 131 { 132 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 133 int ret; 134 135 if (mem->bus.io_reserved_count++) 136 return 0; 137 138 if (!bdev->driver->io_mem_reserve) 139 return 0; 140 141 retry: 142 ret = bdev->driver->io_mem_reserve(bdev, mem); 143 if (ret == -ENOSPC) { 144 ret = ttm_mem_io_evict(man); 145 if (ret == 0) 146 goto retry; 147 } 148 return ret; 149 } 150 151 void ttm_mem_io_free(struct ttm_bo_device *bdev, 152 struct ttm_mem_reg *mem) 153 { 154 if (--mem->bus.io_reserved_count) 155 return; 156 157 if (!bdev->driver->io_mem_free) 158 return; 159 160 bdev->driver->io_mem_free(bdev, mem); 161 } 162 163 int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) 164 { 165 struct ttm_mem_type_manager *man = &bo->bdev->man[bo->mem.mem_type]; 166 struct ttm_mem_reg *mem = &bo->mem; 167 int ret; 168 169 if (mem->bus.io_reserved_vm) 170 return 0; 171 172 ret = ttm_mem_io_reserve(bo->bdev, mem); 173 if (unlikely(ret != 0)) 174 return ret; 175 mem->bus.io_reserved_vm = true; 176 if (man->use_io_reserve_lru) 177 list_add_tail(&bo->io_reserve_lru, 178 &man->io_reserve_lru); 179 return 0; 180 } 181 182 void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) 183 { 184 struct ttm_mem_reg *mem = &bo->mem; 185 186 if (!mem->bus.io_reserved_vm) 187 return; 188 189 mem->bus.io_reserved_vm = false; 190 list_del_init(&bo->io_reserve_lru); 191 ttm_mem_io_free(bo->bdev, mem); 192 } 193 194 static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, 195 struct ttm_mem_reg *mem, 196 void **virtual) 197 { 198 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 199 int ret; 200 void *addr; 201 202 *virtual = NULL; 203 (void) ttm_mem_io_lock(man, false); 204 ret = ttm_mem_io_reserve(bdev, mem); 205 ttm_mem_io_unlock(man); 206 if (ret || !mem->bus.is_iomem) 207 return ret; 208 209 if (mem->bus.addr) { 210 addr = mem->bus.addr; 211 } else { 212 if (mem->placement & TTM_PL_FLAG_WC) 213 addr = ioremap_wc(mem->bus.base + mem->bus.offset, 214 mem->bus.size); 215 else 216 addr = ioremap(mem->bus.base + mem->bus.offset, 217 mem->bus.size); 218 if (!addr) { 219 (void) ttm_mem_io_lock(man, false); 220 ttm_mem_io_free(bdev, mem); 221 ttm_mem_io_unlock(man); 222 return -ENOMEM; 223 } 224 } 225 *virtual = addr; 226 return 0; 227 } 228 229 static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, 230 struct ttm_mem_reg *mem, 231 void *virtual) 232 { 233 struct ttm_mem_type_manager *man; 234 235 man = &bdev->man[mem->mem_type]; 236 237 if (virtual && mem->bus.addr == NULL) 238 iounmap(virtual); 239 (void) ttm_mem_io_lock(man, false); 240 ttm_mem_io_free(bdev, mem); 241 ttm_mem_io_unlock(man); 242 } 243 244 static int ttm_copy_io_page(void *dst, void *src, unsigned long page) 245 { 246 uint32_t *dstP = 247 (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); 248 uint32_t *srcP = 249 (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); 250 251 int i; 252 for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) 253 iowrite32(ioread32(srcP++), dstP++); 254 return 0; 255 } 256 257 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, 258 unsigned long page, 259 pgprot_t prot) 260 { 261 struct page *d = ttm->pages[page]; 262 void *dst; 263 264 if (!d) 265 return -ENOMEM; 266 267 src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); 268 dst = kmap_atomic_prot(d, prot); 269 if (!dst) 270 return -ENOMEM; 271 272 memcpy_fromio(dst, src, PAGE_SIZE); 273 274 kunmap_atomic(dst); 275 276 return 0; 277 } 278 279 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, 280 unsigned long page, 281 pgprot_t prot) 282 { 283 struct page *s = ttm->pages[page]; 284 void *src; 285 286 if (!s) 287 return -ENOMEM; 288 289 dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); 290 src = kmap_atomic_prot(s, prot); 291 if (!src) 292 return -ENOMEM; 293 294 memcpy_toio(dst, src, PAGE_SIZE); 295 296 kunmap_atomic(src); 297 298 return 0; 299 } 300 301 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, 302 struct ttm_operation_ctx *ctx, 303 struct ttm_mem_reg *new_mem) 304 { 305 struct ttm_bo_device *bdev = bo->bdev; 306 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 307 struct ttm_tt *ttm = bo->ttm; 308 struct ttm_mem_reg *old_mem = &bo->mem; 309 struct ttm_mem_reg old_copy = *old_mem; 310 void *old_iomap; 311 void *new_iomap; 312 int ret; 313 unsigned long i; 314 unsigned long page; 315 unsigned long add = 0; 316 int dir; 317 318 ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); 319 if (ret) 320 return ret; 321 322 ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap); 323 if (ret) 324 return ret; 325 ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap); 326 if (ret) 327 goto out; 328 329 /* 330 * Single TTM move. NOP. 331 */ 332 if (old_iomap == NULL && new_iomap == NULL) 333 goto out2; 334 335 /* 336 * Don't move nonexistent data. Clear destination instead. 337 */ 338 if (old_iomap == NULL && 339 (ttm == NULL || (ttm->state == tt_unpopulated && 340 !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) { 341 memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE); 342 goto out2; 343 } 344 345 /* 346 * TTM might be null for moves within the same region. 347 */ 348 if (ttm) { 349 ret = ttm_tt_populate(ttm, ctx); 350 if (ret) 351 goto out1; 352 } 353 354 add = 0; 355 dir = 1; 356 357 if ((old_mem->mem_type == new_mem->mem_type) && 358 (new_mem->start < old_mem->start + old_mem->size)) { 359 dir = -1; 360 add = new_mem->num_pages - 1; 361 } 362 363 for (i = 0; i < new_mem->num_pages; ++i) { 364 page = i * dir + add; 365 if (old_iomap == NULL) { 366 pgprot_t prot = ttm_io_prot(old_mem->placement, 367 PAGE_KERNEL); 368 ret = ttm_copy_ttm_io_page(ttm, new_iomap, page, 369 prot); 370 } else if (new_iomap == NULL) { 371 pgprot_t prot = ttm_io_prot(new_mem->placement, 372 PAGE_KERNEL); 373 ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, 374 prot); 375 } else { 376 ret = ttm_copy_io_page(new_iomap, old_iomap, page); 377 } 378 if (ret) 379 goto out1; 380 } 381 mb(); 382 out2: 383 old_copy = *old_mem; 384 *old_mem = *new_mem; 385 new_mem->mm_node = NULL; 386 387 if (man->flags & TTM_MEMTYPE_FLAG_FIXED) { 388 ttm_tt_destroy(ttm); 389 bo->ttm = NULL; 390 } 391 392 out1: 393 ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); 394 out: 395 ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); 396 397 /* 398 * On error, keep the mm node! 399 */ 400 if (!ret) 401 ttm_bo_mem_put(bo, &old_copy); 402 return ret; 403 } 404 EXPORT_SYMBOL(ttm_bo_move_memcpy); 405 406 static void ttm_transfered_destroy(struct ttm_buffer_object *bo) 407 { 408 struct ttm_transfer_obj *fbo; 409 410 fbo = container_of(bo, struct ttm_transfer_obj, base); 411 ttm_bo_put(fbo->bo); 412 kfree(fbo); 413 } 414 415 /** 416 * ttm_buffer_object_transfer 417 * 418 * @bo: A pointer to a struct ttm_buffer_object. 419 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, 420 * holding the data of @bo with the old placement. 421 * 422 * This is a utility function that may be called after an accelerated move 423 * has been scheduled. A new buffer object is created as a placeholder for 424 * the old data while it's being copied. When that buffer object is idle, 425 * it can be destroyed, releasing the space of the old placement. 426 * Returns: 427 * !0: Failure. 428 */ 429 430 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, 431 struct ttm_buffer_object **new_obj) 432 { 433 struct ttm_transfer_obj *fbo; 434 int ret; 435 436 fbo = kmalloc(sizeof(*fbo), GFP_KERNEL); 437 if (!fbo) 438 return -ENOMEM; 439 440 fbo->base = *bo; 441 fbo->base.mem.placement |= TTM_PL_FLAG_NO_EVICT; 442 443 ttm_bo_get(bo); 444 fbo->bo = bo; 445 446 /** 447 * Fix up members that we shouldn't copy directly: 448 * TODO: Explicit member copy would probably be better here. 449 */ 450 451 atomic_inc(&ttm_bo_glob.bo_count); 452 INIT_LIST_HEAD(&fbo->base.ddestroy); 453 INIT_LIST_HEAD(&fbo->base.lru); 454 INIT_LIST_HEAD(&fbo->base.swap); 455 INIT_LIST_HEAD(&fbo->base.io_reserve_lru); 456 fbo->base.moving = NULL; 457 drm_vma_node_reset(&fbo->base.base.vma_node); 458 459 kref_init(&fbo->base.kref); 460 fbo->base.destroy = &ttm_transfered_destroy; 461 fbo->base.acc_size = 0; 462 if (bo->type != ttm_bo_type_sg) 463 fbo->base.base.resv = &fbo->base.base._resv; 464 465 dma_resv_init(&fbo->base.base._resv); 466 fbo->base.base.dev = NULL; 467 ret = dma_resv_trylock(&fbo->base.base._resv); 468 WARN_ON(!ret); 469 470 *new_obj = &fbo->base; 471 return 0; 472 } 473 474 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) 475 { 476 /* Cached mappings need no adjustment */ 477 if (caching_flags & TTM_PL_FLAG_CACHED) 478 return tmp; 479 480 #if defined(__i386__) || defined(__x86_64__) 481 if (caching_flags & TTM_PL_FLAG_WC) 482 tmp = pgprot_writecombine(tmp); 483 else if (boot_cpu_data.x86 > 3) 484 tmp = pgprot_noncached(tmp); 485 #endif 486 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \ 487 defined(__powerpc__) || defined(__mips__) 488 if (caching_flags & TTM_PL_FLAG_WC) 489 tmp = pgprot_writecombine(tmp); 490 else 491 tmp = pgprot_noncached(tmp); 492 #endif 493 #if defined(__sparc__) 494 tmp = pgprot_noncached(tmp); 495 #endif 496 return tmp; 497 } 498 EXPORT_SYMBOL(ttm_io_prot); 499 500 static int ttm_bo_ioremap(struct ttm_buffer_object *bo, 501 unsigned long offset, 502 unsigned long size, 503 struct ttm_bo_kmap_obj *map) 504 { 505 struct ttm_mem_reg *mem = &bo->mem; 506 507 if (bo->mem.bus.addr) { 508 map->bo_kmap_type = ttm_bo_map_premapped; 509 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); 510 } else { 511 map->bo_kmap_type = ttm_bo_map_iomap; 512 if (mem->placement & TTM_PL_FLAG_WC) 513 map->virtual = ioremap_wc(bo->mem.bus.base + 514 bo->mem.bus.offset + offset, 515 size); 516 else 517 map->virtual = ioremap(bo->mem.bus.base + 518 bo->mem.bus.offset + offset, 519 size); 520 } 521 return (!map->virtual) ? -ENOMEM : 0; 522 } 523 524 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, 525 unsigned long start_page, 526 unsigned long num_pages, 527 struct ttm_bo_kmap_obj *map) 528 { 529 struct ttm_mem_reg *mem = &bo->mem; 530 struct ttm_operation_ctx ctx = { 531 .interruptible = false, 532 .no_wait_gpu = false 533 }; 534 struct ttm_tt *ttm = bo->ttm; 535 pgprot_t prot; 536 int ret; 537 538 BUG_ON(!ttm); 539 540 ret = ttm_tt_populate(ttm, &ctx); 541 if (ret) 542 return ret; 543 544 if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { 545 /* 546 * We're mapping a single page, and the desired 547 * page protection is consistent with the bo. 548 */ 549 550 map->bo_kmap_type = ttm_bo_map_kmap; 551 map->page = ttm->pages[start_page]; 552 map->virtual = kmap(map->page); 553 } else { 554 /* 555 * We need to use vmap to get the desired page protection 556 * or to make the buffer object look contiguous. 557 */ 558 prot = ttm_io_prot(mem->placement, PAGE_KERNEL); 559 map->bo_kmap_type = ttm_bo_map_vmap; 560 map->virtual = vmap(ttm->pages + start_page, num_pages, 561 0, prot); 562 } 563 return (!map->virtual) ? -ENOMEM : 0; 564 } 565 566 int ttm_bo_kmap(struct ttm_buffer_object *bo, 567 unsigned long start_page, unsigned long num_pages, 568 struct ttm_bo_kmap_obj *map) 569 { 570 struct ttm_mem_type_manager *man = 571 &bo->bdev->man[bo->mem.mem_type]; 572 unsigned long offset, size; 573 int ret; 574 575 map->virtual = NULL; 576 map->bo = bo; 577 if (num_pages > bo->num_pages) 578 return -EINVAL; 579 if (start_page > bo->num_pages) 580 return -EINVAL; 581 582 (void) ttm_mem_io_lock(man, false); 583 ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); 584 ttm_mem_io_unlock(man); 585 if (ret) 586 return ret; 587 if (!bo->mem.bus.is_iomem) { 588 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); 589 } else { 590 offset = start_page << PAGE_SHIFT; 591 size = num_pages << PAGE_SHIFT; 592 return ttm_bo_ioremap(bo, offset, size, map); 593 } 594 } 595 EXPORT_SYMBOL(ttm_bo_kmap); 596 597 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) 598 { 599 struct ttm_buffer_object *bo = map->bo; 600 struct ttm_mem_type_manager *man = 601 &bo->bdev->man[bo->mem.mem_type]; 602 603 if (!map->virtual) 604 return; 605 switch (map->bo_kmap_type) { 606 case ttm_bo_map_iomap: 607 iounmap(map->virtual); 608 break; 609 case ttm_bo_map_vmap: 610 vunmap(map->virtual); 611 break; 612 case ttm_bo_map_kmap: 613 kunmap(map->page); 614 break; 615 case ttm_bo_map_premapped: 616 break; 617 default: 618 BUG(); 619 } 620 (void) ttm_mem_io_lock(man, false); 621 ttm_mem_io_free(map->bo->bdev, &map->bo->mem); 622 ttm_mem_io_unlock(man); 623 map->virtual = NULL; 624 map->page = NULL; 625 } 626 EXPORT_SYMBOL(ttm_bo_kunmap); 627 628 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, 629 struct dma_fence *fence, 630 bool evict, 631 struct ttm_mem_reg *new_mem) 632 { 633 struct ttm_bo_device *bdev = bo->bdev; 634 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 635 struct ttm_mem_reg *old_mem = &bo->mem; 636 int ret; 637 struct ttm_buffer_object *ghost_obj; 638 639 dma_resv_add_excl_fence(bo->base.resv, fence); 640 if (evict) { 641 ret = ttm_bo_wait(bo, false, false); 642 if (ret) 643 return ret; 644 645 if (man->flags & TTM_MEMTYPE_FLAG_FIXED) { 646 ttm_tt_destroy(bo->ttm); 647 bo->ttm = NULL; 648 } 649 ttm_bo_free_old_node(bo); 650 } else { 651 /** 652 * This should help pipeline ordinary buffer moves. 653 * 654 * Hang old buffer memory on a new buffer object, 655 * and leave it to be released when the GPU 656 * operation has completed. 657 */ 658 659 dma_fence_put(bo->moving); 660 bo->moving = dma_fence_get(fence); 661 662 ret = ttm_buffer_object_transfer(bo, &ghost_obj); 663 if (ret) 664 return ret; 665 666 dma_resv_add_excl_fence(&ghost_obj->base._resv, fence); 667 668 /** 669 * If we're not moving to fixed memory, the TTM object 670 * needs to stay alive. Otherwhise hang it on the ghost 671 * bo to be unbound and destroyed. 672 */ 673 674 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) 675 ghost_obj->ttm = NULL; 676 else 677 bo->ttm = NULL; 678 679 dma_resv_unlock(&ghost_obj->base._resv); 680 ttm_bo_put(ghost_obj); 681 } 682 683 *old_mem = *new_mem; 684 new_mem->mm_node = NULL; 685 686 return 0; 687 } 688 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); 689 690 int ttm_bo_pipeline_move(struct ttm_buffer_object *bo, 691 struct dma_fence *fence, bool evict, 692 struct ttm_mem_reg *new_mem) 693 { 694 struct ttm_bo_device *bdev = bo->bdev; 695 struct ttm_mem_reg *old_mem = &bo->mem; 696 697 struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type]; 698 struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type]; 699 700 int ret; 701 702 dma_resv_add_excl_fence(bo->base.resv, fence); 703 704 if (!evict) { 705 struct ttm_buffer_object *ghost_obj; 706 707 /** 708 * This should help pipeline ordinary buffer moves. 709 * 710 * Hang old buffer memory on a new buffer object, 711 * and leave it to be released when the GPU 712 * operation has completed. 713 */ 714 715 dma_fence_put(bo->moving); 716 bo->moving = dma_fence_get(fence); 717 718 ret = ttm_buffer_object_transfer(bo, &ghost_obj); 719 if (ret) 720 return ret; 721 722 dma_resv_add_excl_fence(&ghost_obj->base._resv, fence); 723 724 /** 725 * If we're not moving to fixed memory, the TTM object 726 * needs to stay alive. Otherwhise hang it on the ghost 727 * bo to be unbound and destroyed. 728 */ 729 730 if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED)) 731 ghost_obj->ttm = NULL; 732 else 733 bo->ttm = NULL; 734 735 dma_resv_unlock(&ghost_obj->base._resv); 736 ttm_bo_put(ghost_obj); 737 738 } else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) { 739 740 /** 741 * BO doesn't have a TTM we need to bind/unbind. Just remember 742 * this eviction and free up the allocation 743 */ 744 745 spin_lock(&from->move_lock); 746 if (!from->move || dma_fence_is_later(fence, from->move)) { 747 dma_fence_put(from->move); 748 from->move = dma_fence_get(fence); 749 } 750 spin_unlock(&from->move_lock); 751 752 ttm_bo_free_old_node(bo); 753 754 dma_fence_put(bo->moving); 755 bo->moving = dma_fence_get(fence); 756 757 } else { 758 /** 759 * Last resort, wait for the move to be completed. 760 * 761 * Should never happen in pratice. 762 */ 763 764 ret = ttm_bo_wait(bo, false, false); 765 if (ret) 766 return ret; 767 768 if (to->flags & TTM_MEMTYPE_FLAG_FIXED) { 769 ttm_tt_destroy(bo->ttm); 770 bo->ttm = NULL; 771 } 772 ttm_bo_free_old_node(bo); 773 } 774 775 *old_mem = *new_mem; 776 new_mem->mm_node = NULL; 777 778 return 0; 779 } 780 EXPORT_SYMBOL(ttm_bo_pipeline_move); 781 782 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) 783 { 784 struct ttm_buffer_object *ghost; 785 int ret; 786 787 ret = ttm_buffer_object_transfer(bo, &ghost); 788 if (ret) 789 return ret; 790 791 ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv); 792 /* Last resort, wait for the BO to be idle when we are OOM */ 793 if (ret) 794 ttm_bo_wait(bo, false, false); 795 796 memset(&bo->mem, 0, sizeof(bo->mem)); 797 bo->mem.mem_type = TTM_PL_SYSTEM; 798 bo->ttm = NULL; 799 800 dma_resv_unlock(&ghost->base._resv); 801 ttm_bo_put(ghost); 802 803 return 0; 804 } 805