1 /************************************************************************** 2 * 3 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #include <drm/ttm/ttm_bo_driver.h> 32 #include <drm/ttm/ttm_placement.h> 33 #include <drm/drm_vma_manager.h> 34 #include <linux/io.h> 35 #include <linux/highmem.h> 36 #include <linux/wait.h> 37 #include <linux/slab.h> 38 #include <linux/vmalloc.h> 39 #include <linux/module.h> 40 #include <linux/reservation.h> 41 42 void ttm_bo_free_old_node(struct ttm_buffer_object *bo) 43 { 44 ttm_bo_mem_put(bo, &bo->mem); 45 } 46 47 int ttm_bo_move_ttm(struct ttm_buffer_object *bo, 48 struct ttm_operation_ctx *ctx, 49 struct ttm_mem_reg *new_mem) 50 { 51 struct ttm_tt *ttm = bo->ttm; 52 struct ttm_mem_reg *old_mem = &bo->mem; 53 int ret; 54 55 if (old_mem->mem_type != TTM_PL_SYSTEM) { 56 ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); 57 58 if (unlikely(ret != 0)) { 59 if (ret != -ERESTARTSYS) 60 pr_err("Failed to expire sync object before unbinding TTM\n"); 61 return ret; 62 } 63 64 ttm_tt_unbind(ttm); 65 ttm_bo_free_old_node(bo); 66 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, 67 TTM_PL_MASK_MEM); 68 old_mem->mem_type = TTM_PL_SYSTEM; 69 } 70 71 ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); 72 if (unlikely(ret != 0)) 73 return ret; 74 75 if (new_mem->mem_type != TTM_PL_SYSTEM) { 76 ret = ttm_tt_bind(ttm, new_mem, ctx); 77 if (unlikely(ret != 0)) 78 return ret; 79 } 80 81 *old_mem = *new_mem; 82 new_mem->mm_node = NULL; 83 84 return 0; 85 } 86 EXPORT_SYMBOL(ttm_bo_move_ttm); 87 88 int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) 89 { 90 if (likely(man->io_reserve_fastpath)) 91 return 0; 92 93 if (interruptible) 94 return mutex_lock_interruptible(&man->io_reserve_mutex); 95 96 mutex_lock(&man->io_reserve_mutex); 97 return 0; 98 } 99 EXPORT_SYMBOL(ttm_mem_io_lock); 100 101 void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) 102 { 103 if (likely(man->io_reserve_fastpath)) 104 return; 105 106 mutex_unlock(&man->io_reserve_mutex); 107 } 108 EXPORT_SYMBOL(ttm_mem_io_unlock); 109 110 static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) 111 { 112 struct ttm_buffer_object *bo; 113 114 if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru)) 115 return -EAGAIN; 116 117 bo = list_first_entry(&man->io_reserve_lru, 118 struct ttm_buffer_object, 119 io_reserve_lru); 120 list_del_init(&bo->io_reserve_lru); 121 ttm_bo_unmap_virtual_locked(bo); 122 123 return 0; 124 } 125 126 127 int ttm_mem_io_reserve(struct ttm_bo_device *bdev, 128 struct ttm_mem_reg *mem) 129 { 130 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 131 int ret = 0; 132 133 if (!bdev->driver->io_mem_reserve) 134 return 0; 135 if (likely(man->io_reserve_fastpath)) 136 return bdev->driver->io_mem_reserve(bdev, mem); 137 138 if (bdev->driver->io_mem_reserve && 139 mem->bus.io_reserved_count++ == 0) { 140 retry: 141 ret = bdev->driver->io_mem_reserve(bdev, mem); 142 if (ret == -EAGAIN) { 143 ret = ttm_mem_io_evict(man); 144 if (ret == 0) 145 goto retry; 146 } 147 } 148 return ret; 149 } 150 EXPORT_SYMBOL(ttm_mem_io_reserve); 151 152 void ttm_mem_io_free(struct ttm_bo_device *bdev, 153 struct ttm_mem_reg *mem) 154 { 155 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 156 157 if (likely(man->io_reserve_fastpath)) 158 return; 159 160 if (bdev->driver->io_mem_reserve && 161 --mem->bus.io_reserved_count == 0 && 162 bdev->driver->io_mem_free) 163 bdev->driver->io_mem_free(bdev, mem); 164 165 } 166 EXPORT_SYMBOL(ttm_mem_io_free); 167 168 int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) 169 { 170 struct ttm_mem_reg *mem = &bo->mem; 171 int ret; 172 173 if (!mem->bus.io_reserved_vm) { 174 struct ttm_mem_type_manager *man = 175 &bo->bdev->man[mem->mem_type]; 176 177 ret = ttm_mem_io_reserve(bo->bdev, mem); 178 if (unlikely(ret != 0)) 179 return ret; 180 mem->bus.io_reserved_vm = true; 181 if (man->use_io_reserve_lru) 182 list_add_tail(&bo->io_reserve_lru, 183 &man->io_reserve_lru); 184 } 185 return 0; 186 } 187 188 void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) 189 { 190 struct ttm_mem_reg *mem = &bo->mem; 191 192 if (mem->bus.io_reserved_vm) { 193 mem->bus.io_reserved_vm = false; 194 list_del_init(&bo->io_reserve_lru); 195 ttm_mem_io_free(bo->bdev, mem); 196 } 197 } 198 199 static int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 200 void **virtual) 201 { 202 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 203 int ret; 204 void *addr; 205 206 *virtual = NULL; 207 (void) ttm_mem_io_lock(man, false); 208 ret = ttm_mem_io_reserve(bdev, mem); 209 ttm_mem_io_unlock(man); 210 if (ret || !mem->bus.is_iomem) 211 return ret; 212 213 if (mem->bus.addr) { 214 addr = mem->bus.addr; 215 } else { 216 if (mem->placement & TTM_PL_FLAG_WC) 217 addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size); 218 else 219 addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size); 220 if (!addr) { 221 (void) ttm_mem_io_lock(man, false); 222 ttm_mem_io_free(bdev, mem); 223 ttm_mem_io_unlock(man); 224 return -ENOMEM; 225 } 226 } 227 *virtual = addr; 228 return 0; 229 } 230 231 static void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 232 void *virtual) 233 { 234 struct ttm_mem_type_manager *man; 235 236 man = &bdev->man[mem->mem_type]; 237 238 if (virtual && mem->bus.addr == NULL) 239 iounmap(virtual); 240 (void) ttm_mem_io_lock(man, false); 241 ttm_mem_io_free(bdev, mem); 242 ttm_mem_io_unlock(man); 243 } 244 245 static int ttm_copy_io_page(void *dst, void *src, unsigned long page) 246 { 247 uint32_t *dstP = 248 (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); 249 uint32_t *srcP = 250 (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); 251 252 int i; 253 for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) 254 iowrite32(ioread32(srcP++), dstP++); 255 return 0; 256 } 257 258 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, 259 unsigned long page, 260 pgprot_t prot) 261 { 262 struct page *d = ttm->pages[page]; 263 void *dst; 264 265 if (!d) 266 return -ENOMEM; 267 268 src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); 269 270 #ifdef CONFIG_X86 271 dst = kmap_atomic_prot(d, prot); 272 #else 273 if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) 274 dst = vmap(&d, 1, 0, prot); 275 else 276 dst = kmap(d); 277 #endif 278 if (!dst) 279 return -ENOMEM; 280 281 memcpy_fromio(dst, src, PAGE_SIZE); 282 283 #ifdef CONFIG_X86 284 kunmap_atomic(dst); 285 #else 286 if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) 287 vunmap(dst); 288 else 289 kunmap(d); 290 #endif 291 292 return 0; 293 } 294 295 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, 296 unsigned long page, 297 pgprot_t prot) 298 { 299 struct page *s = ttm->pages[page]; 300 void *src; 301 302 if (!s) 303 return -ENOMEM; 304 305 dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); 306 #ifdef CONFIG_X86 307 src = kmap_atomic_prot(s, prot); 308 #else 309 if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) 310 src = vmap(&s, 1, 0, prot); 311 else 312 src = kmap(s); 313 #endif 314 if (!src) 315 return -ENOMEM; 316 317 memcpy_toio(dst, src, PAGE_SIZE); 318 319 #ifdef CONFIG_X86 320 kunmap_atomic(src); 321 #else 322 if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) 323 vunmap(src); 324 else 325 kunmap(s); 326 #endif 327 328 return 0; 329 } 330 331 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, 332 struct ttm_operation_ctx *ctx, 333 struct ttm_mem_reg *new_mem) 334 { 335 struct ttm_bo_device *bdev = bo->bdev; 336 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 337 struct ttm_tt *ttm = bo->ttm; 338 struct ttm_mem_reg *old_mem = &bo->mem; 339 struct ttm_mem_reg old_copy = *old_mem; 340 void *old_iomap; 341 void *new_iomap; 342 int ret; 343 unsigned long i; 344 unsigned long page; 345 unsigned long add = 0; 346 int dir; 347 348 ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); 349 if (ret) 350 return ret; 351 352 ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap); 353 if (ret) 354 return ret; 355 ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap); 356 if (ret) 357 goto out; 358 359 /* 360 * Single TTM move. NOP. 361 */ 362 if (old_iomap == NULL && new_iomap == NULL) 363 goto out2; 364 365 /* 366 * Don't move nonexistent data. Clear destination instead. 367 */ 368 if (old_iomap == NULL && 369 (ttm == NULL || (ttm->state == tt_unpopulated && 370 !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) { 371 memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE); 372 goto out2; 373 } 374 375 /* 376 * TTM might be null for moves within the same region. 377 */ 378 if (ttm) { 379 ret = ttm_tt_populate(ttm, ctx); 380 if (ret) 381 goto out1; 382 } 383 384 add = 0; 385 dir = 1; 386 387 if ((old_mem->mem_type == new_mem->mem_type) && 388 (new_mem->start < old_mem->start + old_mem->size)) { 389 dir = -1; 390 add = new_mem->num_pages - 1; 391 } 392 393 for (i = 0; i < new_mem->num_pages; ++i) { 394 page = i * dir + add; 395 if (old_iomap == NULL) { 396 pgprot_t prot = ttm_io_prot(old_mem->placement, 397 PAGE_KERNEL); 398 ret = ttm_copy_ttm_io_page(ttm, new_iomap, page, 399 prot); 400 } else if (new_iomap == NULL) { 401 pgprot_t prot = ttm_io_prot(new_mem->placement, 402 PAGE_KERNEL); 403 ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, 404 prot); 405 } else { 406 ret = ttm_copy_io_page(new_iomap, old_iomap, page); 407 } 408 if (ret) 409 goto out1; 410 } 411 mb(); 412 out2: 413 old_copy = *old_mem; 414 *old_mem = *new_mem; 415 new_mem->mm_node = NULL; 416 417 if (man->flags & TTM_MEMTYPE_FLAG_FIXED) { 418 ttm_tt_destroy(ttm); 419 bo->ttm = NULL; 420 } 421 422 out1: 423 ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); 424 out: 425 ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); 426 427 /* 428 * On error, keep the mm node! 429 */ 430 if (!ret) 431 ttm_bo_mem_put(bo, &old_copy); 432 return ret; 433 } 434 EXPORT_SYMBOL(ttm_bo_move_memcpy); 435 436 static void ttm_transfered_destroy(struct ttm_buffer_object *bo) 437 { 438 kfree(bo); 439 } 440 441 /** 442 * ttm_buffer_object_transfer 443 * 444 * @bo: A pointer to a struct ttm_buffer_object. 445 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, 446 * holding the data of @bo with the old placement. 447 * 448 * This is a utility function that may be called after an accelerated move 449 * has been scheduled. A new buffer object is created as a placeholder for 450 * the old data while it's being copied. When that buffer object is idle, 451 * it can be destroyed, releasing the space of the old placement. 452 * Returns: 453 * !0: Failure. 454 */ 455 456 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, 457 struct ttm_buffer_object **new_obj) 458 { 459 struct ttm_buffer_object *fbo; 460 int ret; 461 462 fbo = kmalloc(sizeof(*fbo), GFP_KERNEL); 463 if (!fbo) 464 return -ENOMEM; 465 466 *fbo = *bo; 467 468 /** 469 * Fix up members that we shouldn't copy directly: 470 * TODO: Explicit member copy would probably be better here. 471 */ 472 473 atomic_inc(&bo->bdev->glob->bo_count); 474 INIT_LIST_HEAD(&fbo->ddestroy); 475 INIT_LIST_HEAD(&fbo->lru); 476 INIT_LIST_HEAD(&fbo->swap); 477 INIT_LIST_HEAD(&fbo->io_reserve_lru); 478 mutex_init(&fbo->wu_mutex); 479 fbo->moving = NULL; 480 drm_vma_node_reset(&fbo->vma_node); 481 atomic_set(&fbo->cpu_writers, 0); 482 483 kref_init(&fbo->list_kref); 484 kref_init(&fbo->kref); 485 fbo->destroy = &ttm_transfered_destroy; 486 fbo->acc_size = 0; 487 fbo->resv = &fbo->ttm_resv; 488 reservation_object_init(fbo->resv); 489 ret = reservation_object_trylock(fbo->resv); 490 WARN_ON(!ret); 491 492 *new_obj = fbo; 493 return 0; 494 } 495 496 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp) 497 { 498 /* Cached mappings need no adjustment */ 499 if (caching_flags & TTM_PL_FLAG_CACHED) 500 return tmp; 501 502 #if defined(__i386__) || defined(__x86_64__) 503 if (caching_flags & TTM_PL_FLAG_WC) 504 tmp = pgprot_writecombine(tmp); 505 else if (boot_cpu_data.x86 > 3) 506 tmp = pgprot_noncached(tmp); 507 #endif 508 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \ 509 defined(__powerpc__) 510 if (caching_flags & TTM_PL_FLAG_WC) 511 tmp = pgprot_writecombine(tmp); 512 else 513 tmp = pgprot_noncached(tmp); 514 #endif 515 #if defined(__sparc__) || defined(__mips__) 516 tmp = pgprot_noncached(tmp); 517 #endif 518 return tmp; 519 } 520 EXPORT_SYMBOL(ttm_io_prot); 521 522 static int ttm_bo_ioremap(struct ttm_buffer_object *bo, 523 unsigned long offset, 524 unsigned long size, 525 struct ttm_bo_kmap_obj *map) 526 { 527 struct ttm_mem_reg *mem = &bo->mem; 528 529 if (bo->mem.bus.addr) { 530 map->bo_kmap_type = ttm_bo_map_premapped; 531 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); 532 } else { 533 map->bo_kmap_type = ttm_bo_map_iomap; 534 if (mem->placement & TTM_PL_FLAG_WC) 535 map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset, 536 size); 537 else 538 map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset, 539 size); 540 } 541 return (!map->virtual) ? -ENOMEM : 0; 542 } 543 544 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, 545 unsigned long start_page, 546 unsigned long num_pages, 547 struct ttm_bo_kmap_obj *map) 548 { 549 struct ttm_mem_reg *mem = &bo->mem; 550 struct ttm_operation_ctx ctx = { 551 .interruptible = false, 552 .no_wait_gpu = false 553 }; 554 struct ttm_tt *ttm = bo->ttm; 555 pgprot_t prot; 556 int ret; 557 558 BUG_ON(!ttm); 559 560 ret = ttm_tt_populate(ttm, &ctx); 561 if (ret) 562 return ret; 563 564 if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { 565 /* 566 * We're mapping a single page, and the desired 567 * page protection is consistent with the bo. 568 */ 569 570 map->bo_kmap_type = ttm_bo_map_kmap; 571 map->page = ttm->pages[start_page]; 572 map->virtual = kmap(map->page); 573 } else { 574 /* 575 * We need to use vmap to get the desired page protection 576 * or to make the buffer object look contiguous. 577 */ 578 prot = ttm_io_prot(mem->placement, PAGE_KERNEL); 579 map->bo_kmap_type = ttm_bo_map_vmap; 580 map->virtual = vmap(ttm->pages + start_page, num_pages, 581 0, prot); 582 } 583 return (!map->virtual) ? -ENOMEM : 0; 584 } 585 586 int ttm_bo_kmap(struct ttm_buffer_object *bo, 587 unsigned long start_page, unsigned long num_pages, 588 struct ttm_bo_kmap_obj *map) 589 { 590 struct ttm_mem_type_manager *man = 591 &bo->bdev->man[bo->mem.mem_type]; 592 unsigned long offset, size; 593 int ret; 594 595 map->virtual = NULL; 596 map->bo = bo; 597 if (num_pages > bo->num_pages) 598 return -EINVAL; 599 if (start_page > bo->num_pages) 600 return -EINVAL; 601 #if 0 602 if (num_pages > 1 && !capable(CAP_SYS_ADMIN)) 603 return -EPERM; 604 #endif 605 (void) ttm_mem_io_lock(man, false); 606 ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); 607 ttm_mem_io_unlock(man); 608 if (ret) 609 return ret; 610 if (!bo->mem.bus.is_iomem) { 611 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); 612 } else { 613 offset = start_page << PAGE_SHIFT; 614 size = num_pages << PAGE_SHIFT; 615 return ttm_bo_ioremap(bo, offset, size, map); 616 } 617 } 618 EXPORT_SYMBOL(ttm_bo_kmap); 619 620 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) 621 { 622 struct ttm_buffer_object *bo = map->bo; 623 struct ttm_mem_type_manager *man = 624 &bo->bdev->man[bo->mem.mem_type]; 625 626 if (!map->virtual) 627 return; 628 switch (map->bo_kmap_type) { 629 case ttm_bo_map_iomap: 630 iounmap(map->virtual); 631 break; 632 case ttm_bo_map_vmap: 633 vunmap(map->virtual); 634 break; 635 case ttm_bo_map_kmap: 636 kunmap(map->page); 637 break; 638 case ttm_bo_map_premapped: 639 break; 640 default: 641 BUG(); 642 } 643 (void) ttm_mem_io_lock(man, false); 644 ttm_mem_io_free(map->bo->bdev, &map->bo->mem); 645 ttm_mem_io_unlock(man); 646 map->virtual = NULL; 647 map->page = NULL; 648 } 649 EXPORT_SYMBOL(ttm_bo_kunmap); 650 651 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, 652 struct dma_fence *fence, 653 bool evict, 654 struct ttm_mem_reg *new_mem) 655 { 656 struct ttm_bo_device *bdev = bo->bdev; 657 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 658 struct ttm_mem_reg *old_mem = &bo->mem; 659 int ret; 660 struct ttm_buffer_object *ghost_obj; 661 662 reservation_object_add_excl_fence(bo->resv, fence); 663 if (evict) { 664 ret = ttm_bo_wait(bo, false, false); 665 if (ret) 666 return ret; 667 668 if (man->flags & TTM_MEMTYPE_FLAG_FIXED) { 669 ttm_tt_destroy(bo->ttm); 670 bo->ttm = NULL; 671 } 672 ttm_bo_free_old_node(bo); 673 } else { 674 /** 675 * This should help pipeline ordinary buffer moves. 676 * 677 * Hang old buffer memory on a new buffer object, 678 * and leave it to be released when the GPU 679 * operation has completed. 680 */ 681 682 dma_fence_put(bo->moving); 683 bo->moving = dma_fence_get(fence); 684 685 ret = ttm_buffer_object_transfer(bo, &ghost_obj); 686 if (ret) 687 return ret; 688 689 reservation_object_add_excl_fence(ghost_obj->resv, fence); 690 691 /** 692 * If we're not moving to fixed memory, the TTM object 693 * needs to stay alive. Otherwhise hang it on the ghost 694 * bo to be unbound and destroyed. 695 */ 696 697 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) 698 ghost_obj->ttm = NULL; 699 else 700 bo->ttm = NULL; 701 702 ttm_bo_unreserve(ghost_obj); 703 ttm_bo_unref(&ghost_obj); 704 } 705 706 *old_mem = *new_mem; 707 new_mem->mm_node = NULL; 708 709 return 0; 710 } 711 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); 712 713 int ttm_bo_pipeline_move(struct ttm_buffer_object *bo, 714 struct dma_fence *fence, bool evict, 715 struct ttm_mem_reg *new_mem) 716 { 717 struct ttm_bo_device *bdev = bo->bdev; 718 struct ttm_mem_reg *old_mem = &bo->mem; 719 720 struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type]; 721 struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type]; 722 723 int ret; 724 725 reservation_object_add_excl_fence(bo->resv, fence); 726 727 if (!evict) { 728 struct ttm_buffer_object *ghost_obj; 729 730 /** 731 * This should help pipeline ordinary buffer moves. 732 * 733 * Hang old buffer memory on a new buffer object, 734 * and leave it to be released when the GPU 735 * operation has completed. 736 */ 737 738 dma_fence_put(bo->moving); 739 bo->moving = dma_fence_get(fence); 740 741 ret = ttm_buffer_object_transfer(bo, &ghost_obj); 742 if (ret) 743 return ret; 744 745 reservation_object_add_excl_fence(ghost_obj->resv, fence); 746 747 /** 748 * If we're not moving to fixed memory, the TTM object 749 * needs to stay alive. Otherwhise hang it on the ghost 750 * bo to be unbound and destroyed. 751 */ 752 753 if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED)) 754 ghost_obj->ttm = NULL; 755 else 756 bo->ttm = NULL; 757 758 ttm_bo_unreserve(ghost_obj); 759 ttm_bo_unref(&ghost_obj); 760 761 } else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) { 762 763 /** 764 * BO doesn't have a TTM we need to bind/unbind. Just remember 765 * this eviction and free up the allocation 766 */ 767 768 spin_lock(&from->move_lock); 769 if (!from->move || dma_fence_is_later(fence, from->move)) { 770 dma_fence_put(from->move); 771 from->move = dma_fence_get(fence); 772 } 773 spin_unlock(&from->move_lock); 774 775 ttm_bo_free_old_node(bo); 776 777 dma_fence_put(bo->moving); 778 bo->moving = dma_fence_get(fence); 779 780 } else { 781 /** 782 * Last resort, wait for the move to be completed. 783 * 784 * Should never happen in pratice. 785 */ 786 787 ret = ttm_bo_wait(bo, false, false); 788 if (ret) 789 return ret; 790 791 if (to->flags & TTM_MEMTYPE_FLAG_FIXED) { 792 ttm_tt_destroy(bo->ttm); 793 bo->ttm = NULL; 794 } 795 ttm_bo_free_old_node(bo); 796 } 797 798 *old_mem = *new_mem; 799 new_mem->mm_node = NULL; 800 801 return 0; 802 } 803 EXPORT_SYMBOL(ttm_bo_pipeline_move); 804 805 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) 806 { 807 struct ttm_buffer_object *ghost; 808 int ret; 809 810 ret = ttm_buffer_object_transfer(bo, &ghost); 811 if (ret) 812 return ret; 813 814 ret = reservation_object_copy_fences(ghost->resv, bo->resv); 815 /* Last resort, wait for the BO to be idle when we are OOM */ 816 if (ret) 817 ttm_bo_wait(bo, false, false); 818 819 memset(&bo->mem, 0, sizeof(bo->mem)); 820 bo->mem.mem_type = TTM_PL_SYSTEM; 821 bo->ttm = NULL; 822 823 ttm_bo_unreserve(ghost); 824 ttm_bo_unref(&ghost); 825 826 return 0; 827 } 828