1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2 /************************************************************************** 3 * 4 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 25 * USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 **************************************************************************/ 28 /* 29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 30 */ 31 32 #include <drm/ttm/ttm_bo_driver.h> 33 #include <drm/ttm/ttm_placement.h> 34 #include <drm/drm_vma_manager.h> 35 #include <linux/io.h> 36 #include <linux/highmem.h> 37 #include <linux/wait.h> 38 #include <linux/slab.h> 39 #include <linux/vmalloc.h> 40 #include <linux/module.h> 41 #include <linux/dma-resv.h> 42 43 struct ttm_transfer_obj { 44 struct ttm_buffer_object base; 45 struct ttm_buffer_object *bo; 46 }; 47 48 int ttm_mem_io_reserve(struct ttm_bo_device *bdev, 49 struct ttm_resource *mem) 50 { 51 if (mem->bus.offset || mem->bus.addr) 52 return 0; 53 54 mem->bus.is_iomem = false; 55 if (!bdev->driver->io_mem_reserve) 56 return 0; 57 58 return bdev->driver->io_mem_reserve(bdev, mem); 59 } 60 61 void ttm_mem_io_free(struct ttm_bo_device *bdev, 62 struct ttm_resource *mem) 63 { 64 if (!mem->bus.offset && !mem->bus.addr) 65 return; 66 67 if (bdev->driver->io_mem_free) 68 bdev->driver->io_mem_free(bdev, mem); 69 70 mem->bus.offset = 0; 71 mem->bus.addr = NULL; 72 } 73 74 static int ttm_resource_ioremap(struct ttm_bo_device *bdev, 75 struct ttm_resource *mem, 76 void **virtual) 77 { 78 int ret; 79 void *addr; 80 81 *virtual = NULL; 82 ret = ttm_mem_io_reserve(bdev, mem); 83 if (ret || !mem->bus.is_iomem) 84 return ret; 85 86 if (mem->bus.addr) { 87 addr = mem->bus.addr; 88 } else { 89 size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT; 90 91 if (mem->bus.caching == ttm_write_combined) 92 addr = ioremap_wc(mem->bus.offset, bus_size); 93 else 94 addr = ioremap(mem->bus.offset, bus_size); 95 if (!addr) { 96 ttm_mem_io_free(bdev, mem); 97 return -ENOMEM; 98 } 99 } 100 *virtual = addr; 101 return 0; 102 } 103 104 static void ttm_resource_iounmap(struct ttm_bo_device *bdev, 105 struct ttm_resource *mem, 106 void *virtual) 107 { 108 if (virtual && mem->bus.addr == NULL) 109 iounmap(virtual); 110 ttm_mem_io_free(bdev, mem); 111 } 112 113 static int ttm_copy_io_page(void *dst, void *src, unsigned long page) 114 { 115 uint32_t *dstP = 116 (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); 117 uint32_t *srcP = 118 (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); 119 120 int i; 121 for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) 122 iowrite32(ioread32(srcP++), dstP++); 123 return 0; 124 } 125 126 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, 127 unsigned long page, 128 pgprot_t prot) 129 { 130 struct page *d = ttm->pages[page]; 131 void *dst; 132 133 if (!d) 134 return -ENOMEM; 135 136 src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); 137 dst = kmap_atomic_prot(d, prot); 138 if (!dst) 139 return -ENOMEM; 140 141 memcpy_fromio(dst, src, PAGE_SIZE); 142 143 kunmap_atomic(dst); 144 145 return 0; 146 } 147 148 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, 149 unsigned long page, 150 pgprot_t prot) 151 { 152 struct page *s = ttm->pages[page]; 153 void *src; 154 155 if (!s) 156 return -ENOMEM; 157 158 dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); 159 src = kmap_atomic_prot(s, prot); 160 if (!src) 161 return -ENOMEM; 162 163 memcpy_toio(dst, src, PAGE_SIZE); 164 165 kunmap_atomic(src); 166 167 return 0; 168 } 169 170 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, 171 struct ttm_operation_ctx *ctx, 172 struct ttm_resource *new_mem) 173 { 174 struct ttm_bo_device *bdev = bo->bdev; 175 struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); 176 struct ttm_tt *ttm = bo->ttm; 177 struct ttm_resource *old_mem = &bo->mem; 178 struct ttm_resource old_copy = *old_mem; 179 void *old_iomap; 180 void *new_iomap; 181 int ret; 182 unsigned long i; 183 184 ret = ttm_bo_wait_ctx(bo, ctx); 185 if (ret) 186 return ret; 187 188 ret = ttm_resource_ioremap(bdev, old_mem, &old_iomap); 189 if (ret) 190 return ret; 191 ret = ttm_resource_ioremap(bdev, new_mem, &new_iomap); 192 if (ret) 193 goto out; 194 195 /* 196 * Single TTM move. NOP. 197 */ 198 if (old_iomap == NULL && new_iomap == NULL) 199 goto out2; 200 201 /* 202 * Don't move nonexistent data. Clear destination instead. 203 */ 204 if (old_iomap == NULL && 205 (ttm == NULL || (!ttm_tt_is_populated(ttm) && 206 !(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)))) { 207 memset_io(new_iomap, 0, new_mem->num_pages*PAGE_SIZE); 208 goto out2; 209 } 210 211 /* 212 * TTM might be null for moves within the same region. 213 */ 214 if (ttm) { 215 ret = ttm_tt_populate(bdev, ttm, ctx); 216 if (ret) 217 goto out1; 218 } 219 220 for (i = 0; i < new_mem->num_pages; ++i) { 221 if (old_iomap == NULL) { 222 pgprot_t prot = ttm_io_prot(bo, old_mem, PAGE_KERNEL); 223 ret = ttm_copy_ttm_io_page(ttm, new_iomap, i, 224 prot); 225 } else if (new_iomap == NULL) { 226 pgprot_t prot = ttm_io_prot(bo, new_mem, PAGE_KERNEL); 227 ret = ttm_copy_io_ttm_page(ttm, old_iomap, i, 228 prot); 229 } else { 230 ret = ttm_copy_io_page(new_iomap, old_iomap, i); 231 } 232 if (ret) 233 goto out1; 234 } 235 mb(); 236 out2: 237 old_copy = *old_mem; 238 239 ttm_bo_assign_mem(bo, new_mem); 240 241 if (!man->use_tt) 242 ttm_bo_tt_destroy(bo); 243 244 out1: 245 ttm_resource_iounmap(bdev, old_mem, new_iomap); 246 out: 247 ttm_resource_iounmap(bdev, &old_copy, old_iomap); 248 249 /* 250 * On error, keep the mm node! 251 */ 252 if (!ret) 253 ttm_resource_free(bo, &old_copy); 254 return ret; 255 } 256 EXPORT_SYMBOL(ttm_bo_move_memcpy); 257 258 static void ttm_transfered_destroy(struct ttm_buffer_object *bo) 259 { 260 struct ttm_transfer_obj *fbo; 261 262 fbo = container_of(bo, struct ttm_transfer_obj, base); 263 ttm_bo_put(fbo->bo); 264 kfree(fbo); 265 } 266 267 /** 268 * ttm_buffer_object_transfer 269 * 270 * @bo: A pointer to a struct ttm_buffer_object. 271 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, 272 * holding the data of @bo with the old placement. 273 * 274 * This is a utility function that may be called after an accelerated move 275 * has been scheduled. A new buffer object is created as a placeholder for 276 * the old data while it's being copied. When that buffer object is idle, 277 * it can be destroyed, releasing the space of the old placement. 278 * Returns: 279 * !0: Failure. 280 */ 281 282 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, 283 struct ttm_buffer_object **new_obj) 284 { 285 struct ttm_transfer_obj *fbo; 286 int ret; 287 288 fbo = kmalloc(sizeof(*fbo), GFP_KERNEL); 289 if (!fbo) 290 return -ENOMEM; 291 292 fbo->base = *bo; 293 294 ttm_bo_get(bo); 295 fbo->bo = bo; 296 297 /** 298 * Fix up members that we shouldn't copy directly: 299 * TODO: Explicit member copy would probably be better here. 300 */ 301 302 atomic_inc(&ttm_bo_glob.bo_count); 303 INIT_LIST_HEAD(&fbo->base.ddestroy); 304 INIT_LIST_HEAD(&fbo->base.lru); 305 INIT_LIST_HEAD(&fbo->base.swap); 306 fbo->base.moving = NULL; 307 drm_vma_node_reset(&fbo->base.base.vma_node); 308 309 kref_init(&fbo->base.kref); 310 fbo->base.destroy = &ttm_transfered_destroy; 311 fbo->base.acc_size = 0; 312 fbo->base.pin_count = 1; 313 if (bo->type != ttm_bo_type_sg) 314 fbo->base.base.resv = &fbo->base.base._resv; 315 316 dma_resv_init(&fbo->base.base._resv); 317 fbo->base.base.dev = NULL; 318 ret = dma_resv_trylock(&fbo->base.base._resv); 319 WARN_ON(!ret); 320 321 *new_obj = &fbo->base; 322 return 0; 323 } 324 325 pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res, 326 pgprot_t tmp) 327 { 328 struct ttm_resource_manager *man; 329 enum ttm_caching caching; 330 331 man = ttm_manager_type(bo->bdev, res->mem_type); 332 caching = man->use_tt ? bo->ttm->caching : res->bus.caching; 333 334 /* Cached mappings need no adjustment */ 335 if (caching == ttm_cached) 336 return tmp; 337 338 #if defined(__i386__) || defined(__x86_64__) 339 if (caching == ttm_write_combined) 340 tmp = pgprot_writecombine(tmp); 341 else if (boot_cpu_data.x86 > 3) 342 tmp = pgprot_noncached(tmp); 343 #endif 344 #if defined(__ia64__) || defined(__arm__) || defined(__aarch64__) || \ 345 defined(__powerpc__) || defined(__mips__) 346 if (caching == ttm_write_combined) 347 tmp = pgprot_writecombine(tmp); 348 else 349 tmp = pgprot_noncached(tmp); 350 #endif 351 #if defined(__sparc__) 352 tmp = pgprot_noncached(tmp); 353 #endif 354 return tmp; 355 } 356 EXPORT_SYMBOL(ttm_io_prot); 357 358 static int ttm_bo_ioremap(struct ttm_buffer_object *bo, 359 unsigned long offset, 360 unsigned long size, 361 struct ttm_bo_kmap_obj *map) 362 { 363 struct ttm_resource *mem = &bo->mem; 364 365 if (bo->mem.bus.addr) { 366 map->bo_kmap_type = ttm_bo_map_premapped; 367 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); 368 } else { 369 map->bo_kmap_type = ttm_bo_map_iomap; 370 if (mem->bus.caching == ttm_write_combined) 371 map->virtual = ioremap_wc(bo->mem.bus.offset + offset, 372 size); 373 else 374 map->virtual = ioremap(bo->mem.bus.offset + offset, 375 size); 376 } 377 return (!map->virtual) ? -ENOMEM : 0; 378 } 379 380 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, 381 unsigned long start_page, 382 unsigned long num_pages, 383 struct ttm_bo_kmap_obj *map) 384 { 385 struct ttm_resource *mem = &bo->mem; 386 struct ttm_operation_ctx ctx = { 387 .interruptible = false, 388 .no_wait_gpu = false 389 }; 390 struct ttm_tt *ttm = bo->ttm; 391 pgprot_t prot; 392 int ret; 393 394 BUG_ON(!ttm); 395 396 ret = ttm_tt_populate(bo->bdev, ttm, &ctx); 397 if (ret) 398 return ret; 399 400 if (num_pages == 1 && ttm->caching == ttm_cached) { 401 /* 402 * We're mapping a single page, and the desired 403 * page protection is consistent with the bo. 404 */ 405 406 map->bo_kmap_type = ttm_bo_map_kmap; 407 map->page = ttm->pages[start_page]; 408 map->virtual = kmap(map->page); 409 } else { 410 /* 411 * We need to use vmap to get the desired page protection 412 * or to make the buffer object look contiguous. 413 */ 414 prot = ttm_io_prot(bo, mem, PAGE_KERNEL); 415 map->bo_kmap_type = ttm_bo_map_vmap; 416 map->virtual = vmap(ttm->pages + start_page, num_pages, 417 0, prot); 418 } 419 return (!map->virtual) ? -ENOMEM : 0; 420 } 421 422 int ttm_bo_kmap(struct ttm_buffer_object *bo, 423 unsigned long start_page, unsigned long num_pages, 424 struct ttm_bo_kmap_obj *map) 425 { 426 unsigned long offset, size; 427 int ret; 428 429 map->virtual = NULL; 430 map->bo = bo; 431 if (num_pages > bo->num_pages) 432 return -EINVAL; 433 if (start_page > bo->num_pages) 434 return -EINVAL; 435 436 ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); 437 if (ret) 438 return ret; 439 if (!bo->mem.bus.is_iomem) { 440 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); 441 } else { 442 offset = start_page << PAGE_SHIFT; 443 size = num_pages << PAGE_SHIFT; 444 return ttm_bo_ioremap(bo, offset, size, map); 445 } 446 } 447 EXPORT_SYMBOL(ttm_bo_kmap); 448 449 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) 450 { 451 if (!map->virtual) 452 return; 453 switch (map->bo_kmap_type) { 454 case ttm_bo_map_iomap: 455 iounmap(map->virtual); 456 break; 457 case ttm_bo_map_vmap: 458 vunmap(map->virtual); 459 break; 460 case ttm_bo_map_kmap: 461 kunmap(map->page); 462 break; 463 case ttm_bo_map_premapped: 464 break; 465 default: 466 BUG(); 467 } 468 ttm_mem_io_free(map->bo->bdev, &map->bo->mem); 469 map->virtual = NULL; 470 map->page = NULL; 471 } 472 EXPORT_SYMBOL(ttm_bo_kunmap); 473 474 static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo, 475 bool dst_use_tt) 476 { 477 int ret; 478 ret = ttm_bo_wait(bo, false, false); 479 if (ret) 480 return ret; 481 482 if (!dst_use_tt) 483 ttm_bo_tt_destroy(bo); 484 ttm_resource_free(bo, &bo->mem); 485 return 0; 486 } 487 488 static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo, 489 struct dma_fence *fence, 490 bool dst_use_tt) 491 { 492 struct ttm_buffer_object *ghost_obj; 493 int ret; 494 495 /** 496 * This should help pipeline ordinary buffer moves. 497 * 498 * Hang old buffer memory on a new buffer object, 499 * and leave it to be released when the GPU 500 * operation has completed. 501 */ 502 503 dma_fence_put(bo->moving); 504 bo->moving = dma_fence_get(fence); 505 506 ret = ttm_buffer_object_transfer(bo, &ghost_obj); 507 if (ret) 508 return ret; 509 510 dma_resv_add_excl_fence(&ghost_obj->base._resv, fence); 511 512 /** 513 * If we're not moving to fixed memory, the TTM object 514 * needs to stay alive. Otherwhise hang it on the ghost 515 * bo to be unbound and destroyed. 516 */ 517 518 if (dst_use_tt) 519 ghost_obj->ttm = NULL; 520 else 521 bo->ttm = NULL; 522 523 dma_resv_unlock(&ghost_obj->base._resv); 524 ttm_bo_put(ghost_obj); 525 return 0; 526 } 527 528 static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo, 529 struct dma_fence *fence) 530 { 531 struct ttm_bo_device *bdev = bo->bdev; 532 struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type); 533 534 /** 535 * BO doesn't have a TTM we need to bind/unbind. Just remember 536 * this eviction and free up the allocation 537 */ 538 spin_lock(&from->move_lock); 539 if (!from->move || dma_fence_is_later(fence, from->move)) { 540 dma_fence_put(from->move); 541 from->move = dma_fence_get(fence); 542 } 543 spin_unlock(&from->move_lock); 544 545 ttm_resource_free(bo, &bo->mem); 546 547 dma_fence_put(bo->moving); 548 bo->moving = dma_fence_get(fence); 549 } 550 551 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, 552 struct dma_fence *fence, 553 bool evict, 554 bool pipeline, 555 struct ttm_resource *new_mem) 556 { 557 struct ttm_bo_device *bdev = bo->bdev; 558 struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type); 559 struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); 560 int ret = 0; 561 562 dma_resv_add_excl_fence(bo->base.resv, fence); 563 if (!evict) 564 ret = ttm_bo_move_to_ghost(bo, fence, man->use_tt); 565 else if (!from->use_tt && pipeline) 566 ttm_bo_move_pipeline_evict(bo, fence); 567 else 568 ret = ttm_bo_wait_free_node(bo, man->use_tt); 569 570 if (ret) 571 return ret; 572 573 ttm_bo_assign_mem(bo, new_mem); 574 575 return 0; 576 } 577 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); 578 579 int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) 580 { 581 struct ttm_buffer_object *ghost; 582 int ret; 583 584 ret = ttm_buffer_object_transfer(bo, &ghost); 585 if (ret) 586 return ret; 587 588 ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv); 589 /* Last resort, wait for the BO to be idle when we are OOM */ 590 if (ret) 591 ttm_bo_wait(bo, false, false); 592 593 memset(&bo->mem, 0, sizeof(bo->mem)); 594 bo->mem.mem_type = TTM_PL_SYSTEM; 595 bo->ttm = NULL; 596 597 dma_resv_unlock(&ghost->base._resv); 598 ttm_bo_put(ghost); 599 600 return 0; 601 } 602