1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2 /************************************************************************** 3 * 4 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 25 * USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 **************************************************************************/ 28 /* 29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 30 */ 31 32 #include <drm/ttm/ttm_bo_driver.h> 33 #include <drm/ttm/ttm_placement.h> 34 #include <drm/drm_vma_manager.h> 35 #include <linux/dma-buf-map.h> 36 #include <linux/io.h> 37 #include <linux/highmem.h> 38 #include <linux/wait.h> 39 #include <linux/slab.h> 40 #include <linux/vmalloc.h> 41 #include <linux/module.h> 42 #include <linux/dma-resv.h> 43 44 struct ttm_transfer_obj { 45 struct ttm_buffer_object base; 46 struct ttm_buffer_object *bo; 47 }; 48 49 int ttm_mem_io_reserve(struct ttm_device *bdev, 50 struct ttm_resource *mem) 51 { 52 if (mem->bus.offset || mem->bus.addr) 53 return 0; 54 55 mem->bus.is_iomem = false; 56 if (!bdev->funcs->io_mem_reserve) 57 return 0; 58 59 return bdev->funcs->io_mem_reserve(bdev, mem); 60 } 61 62 void ttm_mem_io_free(struct ttm_device *bdev, 63 struct ttm_resource *mem) 64 { 65 if (!mem->bus.offset && !mem->bus.addr) 66 return; 67 68 if (bdev->funcs->io_mem_free) 69 bdev->funcs->io_mem_free(bdev, mem); 70 71 mem->bus.offset = 0; 72 mem->bus.addr = NULL; 73 } 74 75 static int ttm_resource_ioremap(struct ttm_device *bdev, 76 struct ttm_resource *mem, 77 void **virtual) 78 { 79 int ret; 80 void *addr; 81 82 *virtual = NULL; 83 ret = ttm_mem_io_reserve(bdev, mem); 84 if (ret || !mem->bus.is_iomem) 85 return ret; 86 87 if (mem->bus.addr) { 88 addr = mem->bus.addr; 89 } else { 90 size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT; 91 92 if (mem->bus.caching == ttm_write_combined) 93 addr = ioremap_wc(mem->bus.offset, bus_size); 94 else 95 addr = ioremap(mem->bus.offset, bus_size); 96 if (!addr) { 97 ttm_mem_io_free(bdev, mem); 98 return -ENOMEM; 99 } 100 } 101 *virtual = addr; 102 return 0; 103 } 104 105 static void ttm_resource_iounmap(struct ttm_device *bdev, 106 struct ttm_resource *mem, 107 void *virtual) 108 { 109 if (virtual && mem->bus.addr == NULL) 110 iounmap(virtual); 111 ttm_mem_io_free(bdev, mem); 112 } 113 114 static int ttm_copy_io_page(void *dst, void *src, unsigned long page) 115 { 116 uint32_t *dstP = 117 (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); 118 uint32_t *srcP = 119 (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); 120 121 int i; 122 for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) 123 iowrite32(ioread32(srcP++), dstP++); 124 return 0; 125 } 126 127 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, 128 unsigned long page, 129 pgprot_t prot) 130 { 131 struct page *d = ttm->pages[page]; 132 void *dst; 133 134 if (!d) 135 return -ENOMEM; 136 137 src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); 138 dst = kmap_atomic_prot(d, prot); 139 if (!dst) 140 return -ENOMEM; 141 142 memcpy_fromio(dst, src, PAGE_SIZE); 143 144 kunmap_atomic(dst); 145 146 return 0; 147 } 148 149 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, 150 unsigned long page, 151 pgprot_t prot) 152 { 153 struct page *s = ttm->pages[page]; 154 void *src; 155 156 if (!s) 157 return -ENOMEM; 158 159 dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); 160 src = kmap_atomic_prot(s, prot); 161 if (!src) 162 return -ENOMEM; 163 164 memcpy_toio(dst, src, PAGE_SIZE); 165 166 kunmap_atomic(src); 167 168 return 0; 169 } 170 171 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, 172 struct ttm_operation_ctx *ctx, 173 struct ttm_resource *new_mem) 174 { 175 struct ttm_device *bdev = bo->bdev; 176 struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); 177 struct ttm_tt *ttm = bo->ttm; 178 struct ttm_resource *old_mem = &bo->mem; 179 struct ttm_resource old_copy = *old_mem; 180 void *old_iomap; 181 void *new_iomap; 182 int ret; 183 unsigned long i; 184 185 ret = ttm_bo_wait_ctx(bo, ctx); 186 if (ret) 187 return ret; 188 189 ret = ttm_resource_ioremap(bdev, old_mem, &old_iomap); 190 if (ret) 191 return ret; 192 ret = ttm_resource_ioremap(bdev, new_mem, &new_iomap); 193 if (ret) 194 goto out; 195 196 /* 197 * Single TTM move. NOP. 198 */ 199 if (old_iomap == NULL && new_iomap == NULL) 200 goto out2; 201 202 /* 203 * Don't move nonexistent data. Clear destination instead. 204 */ 205 if (old_iomap == NULL && 206 (ttm == NULL || (!ttm_tt_is_populated(ttm) && 207 !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) { 208 memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE); 209 goto out2; 210 } 211 212 /* 213 * TTM might be null for moves within the same region. 214 */ 215 if (ttm) { 216 ret = ttm_tt_populate(bdev, ttm, ctx); 217 if (ret) 218 goto out1; 219 } 220 221 for (i = 0; i < new_mem->num_pages; ++i) { 222 if (old_iomap == NULL) { 223 pgprot_t prot = ttm_io_prot(bo, old_mem, PAGE_KERNEL); 224 ret = ttm_copy_ttm_io_page(ttm, new_iomap, i, 225 prot); 226 } else if (new_iomap == NULL) { 227 pgprot_t prot = ttm_io_prot(bo, new_mem, PAGE_KERNEL); 228 ret = ttm_copy_io_ttm_page(ttm, old_iomap, i, 229 prot); 230 } else { 231 ret = ttm_copy_io_page(new_iomap, old_iomap, i); 232 } 233 if (ret) 234 goto out1; 235 } 236 mb(); 237 out2: 238 old_copy = *old_mem; 239 240 ttm_bo_assign_mem(bo, new_mem); 241 242 if (!man->use_tt) 243 ttm_bo_tt_destroy(bo); 244 245 out1: 246 ttm_resource_iounmap(bdev, old_mem, new_iomap); 247 out: 248 ttm_resource_iounmap(bdev, &old_copy, old_iomap); 249 250 /* 251 * On error, keep the mm node! 252 */ 253 if (!ret) 254 ttm_resource_free(bo, &old_copy); 255 return ret; 256 } 257 EXPORT_SYMBOL(ttm_bo_move_memcpy); 258 259 static void ttm_transfered_destroy(struct ttm_buffer_object *bo) 260 { 261 struct ttm_transfer_obj *fbo; 262 263 fbo = container_of(bo, struct ttm_transfer_obj, base); 264 ttm_bo_put(fbo->bo); 265 kfree(fbo); 266 } 267 268 /** 269 * ttm_buffer_object_transfer 270 * 271 * @bo: A pointer to a struct ttm_buffer_object. 272 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, 273 * holding the data of @bo with the old placement. 274 * 275 * This is a utility function that may be called after an accelerated move 276 * has been scheduled. A new buffer object is created as a placeholder for 277 * the old data while it's being copied. When that buffer object is idle, 278 * it can be destroyed, releasing the space of the old placement. 279 * Returns: 280 * !0: Failure. 281 */ 282 283 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, 284 struct ttm_buffer_object **new_obj) 285 { 286 struct ttm_transfer_obj *fbo; 287 int ret; 288 289 fbo = kmalloc(sizeof(*fbo), GFP_KERNEL); 290 if (!fbo) 291 return -ENOMEM; 292 293 fbo->base = *bo; 294 295 ttm_bo_get(bo); 296 fbo->bo = bo; 297 298 /** 299 * Fix up members that we shouldn't copy directly: 300 * TODO: Explicit member copy would probably be better here. 301 */ 302 303 atomic_inc(&ttm_glob.bo_count); 304 INIT_LIST_HEAD(&fbo->base.ddestroy); 305 INIT_LIST_HEAD(&fbo->base.lru); 306 INIT_LIST_HEAD(&fbo->base.swap); 307 fbo->base.moving = NULL; 308 drm_vma_node_reset(&fbo->base.base.vma_node); 309 310 kref_init(&fbo->base.kref); 311 fbo->base.destroy = &ttm_transfered_destroy; 312 fbo->base.pin_count = 0; 313 if (bo->type != ttm_bo_type_sg) 314 fbo->base.base.resv = &fbo->base.base._resv; 315 316 dma_resv_init(&fbo->base.base._resv); 317 fbo->base.base.dev = NULL; 318 ret = dma_resv_trylock(&fbo->base.base._resv); 319 WARN_ON(!ret); 320 321 ttm_bo_move_to_lru_tail_unlocked(&fbo->base); 322 323 *new_obj = &fbo->base; 324 return 0; 325 } 326 327 pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res, 328 pgprot_t tmp) 329 { 330 struct ttm_resource_manager *man; 331 enum ttm_caching caching; 332 333 man = ttm_manager_type(bo->bdev, res->mem_type); 334 caching = man->use_tt ? bo->ttm->caching : res->bus.caching; 335 336 /* Cached mappings need no adjustment */ 337 if (caching == ttm_cached) 338 return tmp; 339 340 #if defined(__i386__) || defined(__x86_64__) 341 if (caching == ttm_write_combined) 342 tmp = pgprot_writecombine(tmp); 343 else if (boot_cpu_data.x86 > 3) 344 tmp = pgprot_noncached(tmp); 345 #endif 346 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \ 347 defined(__powerpc__) || defined(__mips__) 348 if (caching == ttm_write_combined) 349 tmp = pgprot_writecombine(tmp); 350 else 351 tmp = pgprot_noncached(tmp); 352 #endif 353 #if defined(__sparc__) 354 tmp = pgprot_noncached(tmp); 355 #endif 356 return tmp; 357 } 358 EXPORT_SYMBOL(ttm_io_prot); 359 360 static int ttm_bo_ioremap(struct ttm_buffer_object *bo, 361 unsigned long offset, 362 unsigned long size, 363 struct ttm_bo_kmap_obj *map) 364 { 365 struct ttm_resource *mem = &bo->mem; 366 367 if (bo->mem.bus.addr) { 368 map->bo_kmap_type = ttm_bo_map_premapped; 369 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); 370 } else { 371 map->bo_kmap_type = ttm_bo_map_iomap; 372 if (mem->bus.caching == ttm_write_combined) 373 map->virtual = ioremap_wc(bo->mem.bus.offset + offset, 374 size); 375 else 376 map->virtual = ioremap(bo->mem.bus.offset + offset, 377 size); 378 } 379 return (!map->virtual) ? -ENOMEM : 0; 380 } 381 382 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, 383 unsigned long start_page, 384 unsigned long num_pages, 385 struct ttm_bo_kmap_obj *map) 386 { 387 struct ttm_resource *mem = &bo->mem; 388 struct ttm_operation_ctx ctx = { 389 .interruptible = false, 390 .no_wait_gpu = false 391 }; 392 struct ttm_tt *ttm = bo->ttm; 393 pgprot_t prot; 394 int ret; 395 396 BUG_ON(!ttm); 397 398 ret = ttm_tt_populate(bo->bdev, ttm, &ctx); 399 if (ret) 400 return ret; 401 402 if (num_pages == 1 && ttm->caching == ttm_cached) { 403 /* 404 * We're mapping a single page, and the desired 405 * page protection is consistent with the bo. 406 */ 407 408 map->bo_kmap_type = ttm_bo_map_kmap; 409 map->page = ttm->pages[start_page]; 410 map->virtual = kmap(map->page); 411 } else { 412 /* 413 * We need to use vmap to get the desired page protection 414 * or to make the buffer object look contiguous. 415 */ 416 prot = ttm_io_prot(bo, mem, PAGE_KERNEL); 417 map->bo_kmap_type = ttm_bo_map_vmap; 418 map->virtual = vmap(ttm->pages + start_page, num_pages, 419 0, prot); 420 } 421 return (!map->virtual) ? -ENOMEM : 0; 422 } 423 424 int ttm_bo_kmap(struct ttm_buffer_object *bo, 425 unsigned long start_page, unsigned long num_pages, 426 struct ttm_bo_kmap_obj *map) 427 { 428 unsigned long offset, size; 429 int ret; 430 431 map->virtual = NULL; 432 map->bo = bo; 433 if (num_pages > bo->mem.num_pages) 434 return -EINVAL; 435 if ((start_page + num_pages) > bo->mem.num_pages) 436 return -EINVAL; 437 438 ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); 439 if (ret) 440 return ret; 441 if (!bo->mem.bus.is_iomem) { 442 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); 443 } else { 444 offset = start_page << PAGE_SHIFT; 445 size = num_pages << PAGE_SHIFT; 446 return ttm_bo_ioremap(bo, offset, size, map); 447 } 448 } 449 EXPORT_SYMBOL(ttm_bo_kmap); 450 451 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) 452 { 453 if (!map->virtual) 454 return; 455 switch (map->bo_kmap_type) { 456 case ttm_bo_map_iomap: 457 iounmap(map->virtual); 458 break; 459 case ttm_bo_map_vmap: 460 vunmap(map->virtual); 461 break; 462 case ttm_bo_map_kmap: 463 kunmap(map->page); 464 break; 465 case ttm_bo_map_premapped: 466 break; 467 default: 468 BUG(); 469 } 470 ttm_mem_io_free(map->bo->bdev, &map->bo->mem); 471 map->virtual = NULL; 472 map->page = NULL; 473 } 474 EXPORT_SYMBOL(ttm_bo_kunmap); 475 476 int ttm_bo_vmap(struct ttm_buffer_object *bo, struct dma_buf_map *map) 477 { 478 struct ttm_resource *mem = &bo->mem; 479 int ret; 480 481 ret = ttm_mem_io_reserve(bo->bdev, mem); 482 if (ret) 483 return ret; 484 485 if (mem->bus.is_iomem) { 486 void __iomem *vaddr_iomem; 487 488 if (mem->bus.addr) 489 vaddr_iomem = (void __iomem *)mem->bus.addr; 490 else if (mem->bus.caching == ttm_write_combined) 491 vaddr_iomem = ioremap_wc(mem->bus.offset, 492 bo->base.size); 493 else 494 vaddr_iomem = ioremap(mem->bus.offset, bo->base.size); 495 496 if (!vaddr_iomem) 497 return -ENOMEM; 498 499 dma_buf_map_set_vaddr_iomem(map, vaddr_iomem); 500 501 } else { 502 struct ttm_operation_ctx ctx = { 503 .interruptible = false, 504 .no_wait_gpu = false 505 }; 506 struct ttm_tt *ttm = bo->ttm; 507 pgprot_t prot; 508 void *vaddr; 509 510 ret = ttm_tt_populate(bo->bdev, ttm, &ctx); 511 if (ret) 512 return ret; 513 514 /* 515 * We need to use vmap to get the desired page protection 516 * or to make the buffer object look contiguous. 517 */ 518 prot = ttm_io_prot(bo, mem, PAGE_KERNEL); 519 vaddr = vmap(ttm->pages, ttm->num_pages, 0, prot); 520 if (!vaddr) 521 return -ENOMEM; 522 523 dma_buf_map_set_vaddr(map, vaddr); 524 } 525 526 return 0; 527 } 528 EXPORT_SYMBOL(ttm_bo_vmap); 529 530 void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct dma_buf_map *map) 531 { 532 struct ttm_resource *mem = &bo->mem; 533 534 if (dma_buf_map_is_null(map)) 535 return; 536 537 if (!map->is_iomem) 538 vunmap(map->vaddr); 539 else if (!mem->bus.addr) 540 iounmap(map->vaddr_iomem); 541 dma_buf_map_clear(map); 542 543 ttm_mem_io_free(bo->bdev, &bo->mem); 544 } 545 EXPORT_SYMBOL(ttm_bo_vunmap); 546 547 static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo, 548 bool dst_use_tt) 549 { 550 int ret; 551 ret = ttm_bo_wait(bo, false, false); 552 if (ret) 553 return ret; 554 555 if (!dst_use_tt) 556 ttm_bo_tt_destroy(bo); 557 ttm_resource_free(bo, &bo->mem); 558 return 0; 559 } 560 561 static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo, 562 struct dma_fence *fence, 563 bool dst_use_tt) 564 { 565 struct ttm_buffer_object *ghost_obj; 566 int ret; 567 568 /** 569 * This should help pipeline ordinary buffer moves. 570 * 571 * Hang old buffer memory on a new buffer object, 572 * and leave it to be released when the GPU 573 * operation has completed. 574 */ 575 576 dma_fence_put(bo->moving); 577 bo->moving = dma_fence_get(fence); 578 579 ret = ttm_buffer_object_transfer(bo, &ghost_obj); 580 if (ret) 581 return ret; 582 583 dma_resv_add_excl_fence(&ghost_obj->base._resv, fence); 584 585 /** 586 * If we're not moving to fixed memory, the TTM object 587 * needs to stay alive. Otherwhise hang it on the ghost 588 * bo to be unbound and destroyed. 589 */ 590 591 if (dst_use_tt) 592 ghost_obj->ttm = NULL; 593 else 594 bo->ttm = NULL; 595 596 dma_resv_unlock(&ghost_obj->base._resv); 597 ttm_bo_put(ghost_obj); 598 return 0; 599 } 600 601 static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo, 602 struct dma_fence *fence) 603 { 604 struct ttm_device *bdev = bo->bdev; 605 struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type); 606 607 /** 608 * BO doesn't have a TTM we need to bind/unbind. Just remember 609 * this eviction and free up the allocation 610 */ 611 spin_lock(&from->move_lock); 612 if (!from->move || dma_fence_is_later(fence, from->move)) { 613 dma_fence_put(from->move); 614 from->move = dma_fence_get(fence); 615 } 616 spin_unlock(&from->move_lock); 617 618 ttm_resource_free(bo, &bo->mem); 619 620 dma_fence_put(bo->moving); 621 bo->moving = dma_fence_get(fence); 622 } 623 624 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, 625 struct dma_fence *fence, 626 bool evict, 627 bool pipeline, 628 struct ttm_resource *new_mem) 629 { 630 struct ttm_device *bdev = bo->bdev; 631 struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type); 632 struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); 633 int ret = 0; 634 635 dma_resv_add_excl_fence(bo->base.resv, fence); 636 if (!evict) 637 ret = ttm_bo_move_to_ghost(bo, fence, man->use_tt); 638 else if (!from->use_tt && pipeline) 639 ttm_bo_move_pipeline_evict(bo, fence); 640 else 641 ret = ttm_bo_wait_free_node(bo, man->use_tt); 642 643 if (ret) 644 return ret; 645 646 ttm_bo_assign_mem(bo, new_mem); 647 648 return 0; 649 } 650 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); 651 652 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) 653 { 654 struct ttm_buffer_object *ghost; 655 int ret; 656 657 ret = ttm_buffer_object_transfer(bo, &ghost); 658 if (ret) 659 return ret; 660 661 ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv); 662 /* Last resort, wait for the BO to be idle when we are OOM */ 663 if (ret) 664 ttm_bo_wait(bo, false, false); 665 666 memset(&bo->mem, 0, sizeof(bo->mem)); 667 bo->mem.mem_type = TTM_PL_SYSTEM; 668 bo->ttm = NULL; 669 670 dma_resv_unlock(&ghost->base._resv); 671 ttm_bo_put(ghost); 672 673 return 0; 674 } 675