1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 33 #include <linux/dma-mapping.h> 34 #include <linux/pagemap.h> 35 #include <linux/pci.h> 36 #include <linux/seq_file.h> 37 #include <linux/slab.h> 38 #include <linux/swap.h> 39 #include <linux/swiotlb.h> 40 41 #include <drm/drm_agpsupport.h> 42 #include <drm/drm_device.h> 43 #include <drm/drm_file.h> 44 #include <drm/drm_prime.h> 45 #include <drm/radeon_drm.h> 46 #include <drm/ttm/ttm_bo_api.h> 47 #include <drm/ttm/ttm_bo_driver.h> 48 #include <drm/ttm/ttm_placement.h> 49 50 #include "radeon_reg.h" 51 #include "radeon.h" 52 #include "radeon_ttm.h" 53 54 static void radeon_ttm_debugfs_init(struct radeon_device *rdev); 55 56 static int radeon_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, 57 struct ttm_resource *bo_mem); 58 static void radeon_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm); 59 60 struct radeon_device *radeon_get_rdev(struct ttm_device *bdev) 61 { 62 struct radeon_mman *mman; 63 struct radeon_device *rdev; 64 65 mman = container_of(bdev, struct radeon_mman, bdev); 66 rdev = container_of(mman, struct radeon_device, mman); 67 return rdev; 68 } 69 70 static int radeon_ttm_init_vram(struct radeon_device *rdev) 71 { 72 return ttm_range_man_init(&rdev->mman.bdev, TTM_PL_VRAM, 73 false, rdev->mc.real_vram_size >> PAGE_SHIFT); 74 } 75 76 static int radeon_ttm_init_gtt(struct radeon_device *rdev) 77 { 78 return ttm_range_man_init(&rdev->mman.bdev, TTM_PL_TT, 79 true, rdev->mc.gtt_size >> PAGE_SHIFT); 80 } 81 82 static void radeon_evict_flags(struct ttm_buffer_object *bo, 83 struct ttm_placement *placement) 84 { 85 static const struct ttm_place placements = { 86 .fpfn = 0, 87 .lpfn = 0, 88 .mem_type = TTM_PL_SYSTEM, 89 .flags = 0 90 }; 91 92 struct radeon_bo *rbo; 93 94 if (!radeon_ttm_bo_is_radeon_bo(bo)) { 95 placement->placement = &placements; 96 placement->busy_placement = &placements; 97 placement->num_placement = 1; 98 placement->num_busy_placement = 1; 99 return; 100 } 101 rbo = container_of(bo, struct radeon_bo, tbo); 102 switch (bo->mem.mem_type) { 103 case TTM_PL_VRAM: 104 if (rbo->rdev->ring[radeon_copy_ring_index(rbo->rdev)].ready == false) 105 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 106 else if (rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size && 107 bo->mem.start < (rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT)) { 108 unsigned fpfn = rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; 109 int i; 110 111 /* Try evicting to the CPU inaccessible part of VRAM 112 * first, but only set GTT as busy placement, so this 113 * BO will be evicted to GTT rather than causing other 114 * BOs to be evicted from VRAM 115 */ 116 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM | 117 RADEON_GEM_DOMAIN_GTT); 118 rbo->placement.num_busy_placement = 0; 119 for (i = 0; i < rbo->placement.num_placement; i++) { 120 if (rbo->placements[i].mem_type == TTM_PL_VRAM) { 121 if (rbo->placements[i].fpfn < fpfn) 122 rbo->placements[i].fpfn = fpfn; 123 } else { 124 rbo->placement.busy_placement = 125 &rbo->placements[i]; 126 rbo->placement.num_busy_placement = 1; 127 } 128 } 129 } else 130 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); 131 break; 132 case TTM_PL_TT: 133 default: 134 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 135 } 136 *placement = rbo->placement; 137 } 138 139 static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp) 140 { 141 struct radeon_bo *rbo = container_of(bo, struct radeon_bo, tbo); 142 struct radeon_device *rdev = radeon_get_rdev(bo->bdev); 143 144 if (radeon_ttm_tt_has_userptr(rdev, bo->ttm)) 145 return -EPERM; 146 return drm_vma_node_verify_access(&rbo->tbo.base.vma_node, 147 filp->private_data); 148 } 149 150 static int radeon_move_blit(struct ttm_buffer_object *bo, 151 bool evict, 152 struct ttm_resource *new_mem, 153 struct ttm_resource *old_mem) 154 { 155 struct radeon_device *rdev; 156 uint64_t old_start, new_start; 157 struct radeon_fence *fence; 158 unsigned num_pages; 159 int r, ridx; 160 161 rdev = radeon_get_rdev(bo->bdev); 162 ridx = radeon_copy_ring_index(rdev); 163 old_start = (u64)old_mem->start << PAGE_SHIFT; 164 new_start = (u64)new_mem->start << PAGE_SHIFT; 165 166 switch (old_mem->mem_type) { 167 case TTM_PL_VRAM: 168 old_start += rdev->mc.vram_start; 169 break; 170 case TTM_PL_TT: 171 old_start += rdev->mc.gtt_start; 172 break; 173 default: 174 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 175 return -EINVAL; 176 } 177 switch (new_mem->mem_type) { 178 case TTM_PL_VRAM: 179 new_start += rdev->mc.vram_start; 180 break; 181 case TTM_PL_TT: 182 new_start += rdev->mc.gtt_start; 183 break; 184 default: 185 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 186 return -EINVAL; 187 } 188 if (!rdev->ring[ridx].ready) { 189 DRM_ERROR("Trying to move memory with ring turned off.\n"); 190 return -EINVAL; 191 } 192 193 BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); 194 195 num_pages = new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 196 fence = radeon_copy(rdev, old_start, new_start, num_pages, bo->base.resv); 197 if (IS_ERR(fence)) 198 return PTR_ERR(fence); 199 200 r = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, false, new_mem); 201 radeon_fence_unref(&fence); 202 return r; 203 } 204 205 static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, 206 struct ttm_operation_ctx *ctx, 207 struct ttm_resource *new_mem, 208 struct ttm_place *hop) 209 { 210 struct radeon_device *rdev; 211 struct radeon_bo *rbo; 212 struct ttm_resource *old_mem = &bo->mem; 213 int r; 214 215 if (new_mem->mem_type == TTM_PL_TT) { 216 r = radeon_ttm_tt_bind(bo->bdev, bo->ttm, new_mem); 217 if (r) 218 return r; 219 } 220 221 r = ttm_bo_wait_ctx(bo, ctx); 222 if (r) 223 return r; 224 225 /* Can't move a pinned BO */ 226 rbo = container_of(bo, struct radeon_bo, tbo); 227 if (WARN_ON_ONCE(rbo->tbo.pin_count > 0)) 228 return -EINVAL; 229 230 rdev = radeon_get_rdev(bo->bdev); 231 if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 232 ttm_bo_move_null(bo, new_mem); 233 goto out; 234 } 235 if (old_mem->mem_type == TTM_PL_SYSTEM && 236 new_mem->mem_type == TTM_PL_TT) { 237 ttm_bo_move_null(bo, new_mem); 238 goto out; 239 } 240 241 if (old_mem->mem_type == TTM_PL_TT && 242 new_mem->mem_type == TTM_PL_SYSTEM) { 243 radeon_ttm_tt_unbind(bo->bdev, bo->ttm); 244 ttm_resource_free(bo, &bo->mem); 245 ttm_bo_assign_mem(bo, new_mem); 246 goto out; 247 } 248 if (rdev->ring[radeon_copy_ring_index(rdev)].ready && 249 rdev->asic->copy.copy != NULL) { 250 if ((old_mem->mem_type == TTM_PL_SYSTEM && 251 new_mem->mem_type == TTM_PL_VRAM) || 252 (old_mem->mem_type == TTM_PL_VRAM && 253 new_mem->mem_type == TTM_PL_SYSTEM)) { 254 hop->fpfn = 0; 255 hop->lpfn = 0; 256 hop->mem_type = TTM_PL_TT; 257 hop->flags = 0; 258 return -EMULTIHOP; 259 } 260 261 r = radeon_move_blit(bo, evict, new_mem, old_mem); 262 } else { 263 r = -ENODEV; 264 } 265 266 if (r) { 267 r = ttm_bo_move_memcpy(bo, ctx, new_mem); 268 if (r) 269 return r; 270 } 271 272 out: 273 /* update statistics */ 274 atomic64_add(bo->base.size, &rdev->num_bytes_moved); 275 radeon_bo_move_notify(bo, evict, new_mem); 276 return 0; 277 } 278 279 static int radeon_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem) 280 { 281 struct radeon_device *rdev = radeon_get_rdev(bdev); 282 size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT; 283 284 switch (mem->mem_type) { 285 case TTM_PL_SYSTEM: 286 /* system memory */ 287 return 0; 288 case TTM_PL_TT: 289 #if IS_ENABLED(CONFIG_AGP) 290 if (rdev->flags & RADEON_IS_AGP) { 291 /* RADEON_IS_AGP is set only if AGP is active */ 292 mem->bus.offset = (mem->start << PAGE_SHIFT) + 293 rdev->mc.agp_base; 294 mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture; 295 mem->bus.caching = ttm_write_combined; 296 } 297 #endif 298 break; 299 case TTM_PL_VRAM: 300 mem->bus.offset = mem->start << PAGE_SHIFT; 301 /* check if it's visible */ 302 if ((mem->bus.offset + bus_size) > rdev->mc.visible_vram_size) 303 return -EINVAL; 304 mem->bus.offset += rdev->mc.aper_base; 305 mem->bus.is_iomem = true; 306 mem->bus.caching = ttm_write_combined; 307 #ifdef __alpha__ 308 /* 309 * Alpha: use bus.addr to hold the ioremap() return, 310 * so we can modify bus.base below. 311 */ 312 mem->bus.addr = ioremap_wc(mem->bus.offset, bus_size); 313 if (!mem->bus.addr) 314 return -ENOMEM; 315 316 /* 317 * Alpha: Use just the bus offset plus 318 * the hose/domain memory base for bus.base. 319 * It then can be used to build PTEs for VRAM 320 * access, as done in ttm_bo_vm_fault(). 321 */ 322 mem->bus.offset = (mem->bus.offset & 0x0ffffffffUL) + 323 rdev->hose->dense_mem_base; 324 #endif 325 break; 326 default: 327 return -EINVAL; 328 } 329 return 0; 330 } 331 332 /* 333 * TTM backend functions. 334 */ 335 struct radeon_ttm_tt { 336 struct ttm_tt ttm; 337 u64 offset; 338 339 uint64_t userptr; 340 struct mm_struct *usermm; 341 uint32_t userflags; 342 bool bound; 343 }; 344 345 /* prepare the sg table with the user pages */ 346 static int radeon_ttm_tt_pin_userptr(struct ttm_device *bdev, struct ttm_tt *ttm) 347 { 348 struct radeon_device *rdev = radeon_get_rdev(bdev); 349 struct radeon_ttm_tt *gtt = (void *)ttm; 350 unsigned pinned = 0; 351 int r; 352 353 int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 354 enum dma_data_direction direction = write ? 355 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 356 357 if (current->mm != gtt->usermm) 358 return -EPERM; 359 360 if (gtt->userflags & RADEON_GEM_USERPTR_ANONONLY) { 361 /* check that we only pin down anonymous memory 362 to prevent problems with writeback */ 363 unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE; 364 struct vm_area_struct *vma; 365 vma = find_vma(gtt->usermm, gtt->userptr); 366 if (!vma || vma->vm_file || vma->vm_end < end) 367 return -EPERM; 368 } 369 370 do { 371 unsigned num_pages = ttm->num_pages - pinned; 372 uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE; 373 struct page **pages = ttm->pages + pinned; 374 375 r = get_user_pages(userptr, num_pages, write ? FOLL_WRITE : 0, 376 pages, NULL); 377 if (r < 0) 378 goto release_pages; 379 380 pinned += r; 381 382 } while (pinned < ttm->num_pages); 383 384 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, 385 ttm->num_pages << PAGE_SHIFT, 386 GFP_KERNEL); 387 if (r) 388 goto release_sg; 389 390 r = dma_map_sgtable(rdev->dev, ttm->sg, direction, 0); 391 if (r) 392 goto release_sg; 393 394 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, 395 ttm->num_pages); 396 397 return 0; 398 399 release_sg: 400 kfree(ttm->sg); 401 402 release_pages: 403 release_pages(ttm->pages, pinned); 404 return r; 405 } 406 407 static void radeon_ttm_tt_unpin_userptr(struct ttm_device *bdev, struct ttm_tt *ttm) 408 { 409 struct radeon_device *rdev = radeon_get_rdev(bdev); 410 struct radeon_ttm_tt *gtt = (void *)ttm; 411 struct sg_page_iter sg_iter; 412 413 int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 414 enum dma_data_direction direction = write ? 415 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 416 417 /* double check that we don't free the table twice */ 418 if (!ttm->sg || !ttm->sg->sgl) 419 return; 420 421 /* free the sg table and pages again */ 422 dma_unmap_sgtable(rdev->dev, ttm->sg, direction, 0); 423 424 for_each_sgtable_page(ttm->sg, &sg_iter, 0) { 425 struct page *page = sg_page_iter_page(&sg_iter); 426 if (!(gtt->userflags & RADEON_GEM_USERPTR_READONLY)) 427 set_page_dirty(page); 428 429 mark_page_accessed(page); 430 put_page(page); 431 } 432 433 sg_free_table(ttm->sg); 434 } 435 436 static bool radeon_ttm_backend_is_bound(struct ttm_tt *ttm) 437 { 438 struct radeon_ttm_tt *gtt = (void*)ttm; 439 440 return (gtt->bound); 441 } 442 443 static int radeon_ttm_backend_bind(struct ttm_device *bdev, 444 struct ttm_tt *ttm, 445 struct ttm_resource *bo_mem) 446 { 447 struct radeon_ttm_tt *gtt = (void*)ttm; 448 struct radeon_device *rdev = radeon_get_rdev(bdev); 449 uint32_t flags = RADEON_GART_PAGE_VALID | RADEON_GART_PAGE_READ | 450 RADEON_GART_PAGE_WRITE; 451 int r; 452 453 if (gtt->bound) 454 return 0; 455 456 if (gtt->userptr) { 457 radeon_ttm_tt_pin_userptr(bdev, ttm); 458 flags &= ~RADEON_GART_PAGE_WRITE; 459 } 460 461 gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); 462 if (!ttm->num_pages) { 463 WARN(1, "nothing to bind %u pages for mreg %p back %p!\n", 464 ttm->num_pages, bo_mem, ttm); 465 } 466 if (ttm->caching == ttm_cached) 467 flags |= RADEON_GART_PAGE_SNOOP; 468 r = radeon_gart_bind(rdev, gtt->offset, ttm->num_pages, 469 ttm->pages, gtt->ttm.dma_address, flags); 470 if (r) { 471 DRM_ERROR("failed to bind %u pages at 0x%08X\n", 472 ttm->num_pages, (unsigned)gtt->offset); 473 return r; 474 } 475 gtt->bound = true; 476 return 0; 477 } 478 479 static void radeon_ttm_backend_unbind(struct ttm_device *bdev, struct ttm_tt *ttm) 480 { 481 struct radeon_ttm_tt *gtt = (void *)ttm; 482 struct radeon_device *rdev = radeon_get_rdev(bdev); 483 484 if (gtt->userptr) 485 radeon_ttm_tt_unpin_userptr(bdev, ttm); 486 487 if (!gtt->bound) 488 return; 489 490 radeon_gart_unbind(rdev, gtt->offset, ttm->num_pages); 491 492 gtt->bound = false; 493 } 494 495 static void radeon_ttm_backend_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) 496 { 497 struct radeon_ttm_tt *gtt = (void *)ttm; 498 499 radeon_ttm_backend_unbind(bdev, ttm); 500 ttm_tt_destroy_common(bdev, ttm); 501 502 ttm_tt_fini(>t->ttm); 503 kfree(gtt); 504 } 505 506 static struct ttm_tt *radeon_ttm_tt_create(struct ttm_buffer_object *bo, 507 uint32_t page_flags) 508 { 509 struct radeon_ttm_tt *gtt; 510 enum ttm_caching caching; 511 struct radeon_bo *rbo; 512 #if IS_ENABLED(CONFIG_AGP) 513 struct radeon_device *rdev = radeon_get_rdev(bo->bdev); 514 515 if (rdev->flags & RADEON_IS_AGP) { 516 return ttm_agp_tt_create(bo, rdev->ddev->agp->bridge, 517 page_flags); 518 } 519 #endif 520 rbo = container_of(bo, struct radeon_bo, tbo); 521 522 gtt = kzalloc(sizeof(struct radeon_ttm_tt), GFP_KERNEL); 523 if (gtt == NULL) { 524 return NULL; 525 } 526 527 if (rbo->flags & RADEON_GEM_GTT_UC) 528 caching = ttm_uncached; 529 else if (rbo->flags & RADEON_GEM_GTT_WC) 530 caching = ttm_write_combined; 531 else 532 caching = ttm_cached; 533 534 if (ttm_sg_tt_init(>t->ttm, bo, page_flags, caching)) { 535 kfree(gtt); 536 return NULL; 537 } 538 return >t->ttm; 539 } 540 541 static struct radeon_ttm_tt *radeon_ttm_tt_to_gtt(struct radeon_device *rdev, 542 struct ttm_tt *ttm) 543 { 544 #if IS_ENABLED(CONFIG_AGP) 545 if (rdev->flags & RADEON_IS_AGP) 546 return NULL; 547 #endif 548 549 if (!ttm) 550 return NULL; 551 return container_of(ttm, struct radeon_ttm_tt, ttm); 552 } 553 554 static int radeon_ttm_tt_populate(struct ttm_device *bdev, 555 struct ttm_tt *ttm, 556 struct ttm_operation_ctx *ctx) 557 { 558 struct radeon_device *rdev = radeon_get_rdev(bdev); 559 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); 560 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 561 562 if (gtt && gtt->userptr) { 563 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); 564 if (!ttm->sg) 565 return -ENOMEM; 566 567 ttm->page_flags |= TTM_PAGE_FLAG_SG; 568 return 0; 569 } 570 571 if (slave && ttm->sg) { 572 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, 573 ttm->num_pages); 574 return 0; 575 } 576 577 return ttm_pool_alloc(&rdev->mman.bdev.pool, ttm, ctx); 578 } 579 580 static void radeon_ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm) 581 { 582 struct radeon_device *rdev = radeon_get_rdev(bdev); 583 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); 584 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 585 586 if (gtt && gtt->userptr) { 587 kfree(ttm->sg); 588 ttm->page_flags &= ~TTM_PAGE_FLAG_SG; 589 return; 590 } 591 592 if (slave) 593 return; 594 595 return ttm_pool_free(&rdev->mman.bdev.pool, ttm); 596 } 597 598 int radeon_ttm_tt_set_userptr(struct radeon_device *rdev, 599 struct ttm_tt *ttm, uint64_t addr, 600 uint32_t flags) 601 { 602 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); 603 604 if (gtt == NULL) 605 return -EINVAL; 606 607 gtt->userptr = addr; 608 gtt->usermm = current->mm; 609 gtt->userflags = flags; 610 return 0; 611 } 612 613 bool radeon_ttm_tt_is_bound(struct ttm_device *bdev, 614 struct ttm_tt *ttm) 615 { 616 #if IS_ENABLED(CONFIG_AGP) 617 struct radeon_device *rdev = radeon_get_rdev(bdev); 618 if (rdev->flags & RADEON_IS_AGP) 619 return ttm_agp_is_bound(ttm); 620 #endif 621 return radeon_ttm_backend_is_bound(ttm); 622 } 623 624 static int radeon_ttm_tt_bind(struct ttm_device *bdev, 625 struct ttm_tt *ttm, 626 struct ttm_resource *bo_mem) 627 { 628 #if IS_ENABLED(CONFIG_AGP) 629 struct radeon_device *rdev = radeon_get_rdev(bdev); 630 #endif 631 632 if (!bo_mem) 633 return -EINVAL; 634 #if IS_ENABLED(CONFIG_AGP) 635 if (rdev->flags & RADEON_IS_AGP) 636 return ttm_agp_bind(ttm, bo_mem); 637 #endif 638 639 return radeon_ttm_backend_bind(bdev, ttm, bo_mem); 640 } 641 642 static void radeon_ttm_tt_unbind(struct ttm_device *bdev, 643 struct ttm_tt *ttm) 644 { 645 #if IS_ENABLED(CONFIG_AGP) 646 struct radeon_device *rdev = radeon_get_rdev(bdev); 647 648 if (rdev->flags & RADEON_IS_AGP) { 649 ttm_agp_unbind(ttm); 650 return; 651 } 652 #endif 653 radeon_ttm_backend_unbind(bdev, ttm); 654 } 655 656 static void radeon_ttm_tt_destroy(struct ttm_device *bdev, 657 struct ttm_tt *ttm) 658 { 659 #if IS_ENABLED(CONFIG_AGP) 660 struct radeon_device *rdev = radeon_get_rdev(bdev); 661 662 if (rdev->flags & RADEON_IS_AGP) { 663 ttm_agp_unbind(ttm); 664 ttm_tt_destroy_common(bdev, ttm); 665 ttm_agp_destroy(ttm); 666 return; 667 } 668 #endif 669 radeon_ttm_backend_destroy(bdev, ttm); 670 } 671 672 bool radeon_ttm_tt_has_userptr(struct radeon_device *rdev, 673 struct ttm_tt *ttm) 674 { 675 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); 676 677 if (gtt == NULL) 678 return false; 679 680 return !!gtt->userptr; 681 } 682 683 bool radeon_ttm_tt_is_readonly(struct radeon_device *rdev, 684 struct ttm_tt *ttm) 685 { 686 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); 687 688 if (gtt == NULL) 689 return false; 690 691 return !!(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 692 } 693 694 static void 695 radeon_bo_delete_mem_notify(struct ttm_buffer_object *bo) 696 { 697 radeon_bo_move_notify(bo, false, NULL); 698 } 699 700 static struct ttm_device_funcs radeon_bo_driver = { 701 .ttm_tt_create = &radeon_ttm_tt_create, 702 .ttm_tt_populate = &radeon_ttm_tt_populate, 703 .ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate, 704 .ttm_tt_destroy = &radeon_ttm_tt_destroy, 705 .eviction_valuable = ttm_bo_eviction_valuable, 706 .evict_flags = &radeon_evict_flags, 707 .move = &radeon_bo_move, 708 .verify_access = &radeon_verify_access, 709 .delete_mem_notify = &radeon_bo_delete_mem_notify, 710 .io_mem_reserve = &radeon_ttm_io_mem_reserve, 711 }; 712 713 int radeon_ttm_init(struct radeon_device *rdev) 714 { 715 int r; 716 717 /* No others user of address space so set it to 0 */ 718 r = ttm_device_init(&rdev->mman.bdev, &radeon_bo_driver, rdev->dev, 719 rdev->ddev->anon_inode->i_mapping, 720 rdev->ddev->vma_offset_manager, 721 rdev->need_swiotlb, 722 dma_addressing_limited(&rdev->pdev->dev)); 723 if (r) { 724 DRM_ERROR("failed initializing buffer object driver(%d).\n", r); 725 return r; 726 } 727 rdev->mman.initialized = true; 728 729 r = radeon_ttm_init_vram(rdev); 730 if (r) { 731 DRM_ERROR("Failed initializing VRAM heap.\n"); 732 return r; 733 } 734 /* Change the size here instead of the init above so only lpfn is affected */ 735 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 736 737 r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true, 738 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 739 NULL, &rdev->stolen_vga_memory); 740 if (r) { 741 return r; 742 } 743 r = radeon_bo_reserve(rdev->stolen_vga_memory, false); 744 if (r) 745 return r; 746 r = radeon_bo_pin(rdev->stolen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL); 747 radeon_bo_unreserve(rdev->stolen_vga_memory); 748 if (r) { 749 radeon_bo_unref(&rdev->stolen_vga_memory); 750 return r; 751 } 752 DRM_INFO("radeon: %uM of VRAM memory ready\n", 753 (unsigned) (rdev->mc.real_vram_size / (1024 * 1024))); 754 755 r = radeon_ttm_init_gtt(rdev); 756 if (r) { 757 DRM_ERROR("Failed initializing GTT heap.\n"); 758 return r; 759 } 760 DRM_INFO("radeon: %uM of GTT memory ready.\n", 761 (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); 762 763 radeon_ttm_debugfs_init(rdev); 764 765 return 0; 766 } 767 768 void radeon_ttm_fini(struct radeon_device *rdev) 769 { 770 int r; 771 772 if (!rdev->mman.initialized) 773 return; 774 775 if (rdev->stolen_vga_memory) { 776 r = radeon_bo_reserve(rdev->stolen_vga_memory, false); 777 if (r == 0) { 778 radeon_bo_unpin(rdev->stolen_vga_memory); 779 radeon_bo_unreserve(rdev->stolen_vga_memory); 780 } 781 radeon_bo_unref(&rdev->stolen_vga_memory); 782 } 783 ttm_range_man_fini(&rdev->mman.bdev, TTM_PL_VRAM); 784 ttm_range_man_fini(&rdev->mman.bdev, TTM_PL_TT); 785 ttm_device_fini(&rdev->mman.bdev); 786 radeon_gart_fini(rdev); 787 rdev->mman.initialized = false; 788 DRM_INFO("radeon: ttm finalized\n"); 789 } 790 791 /* this should only be called at bootup or when userspace 792 * isn't running */ 793 void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size) 794 { 795 struct ttm_resource_manager *man; 796 797 if (!rdev->mman.initialized) 798 return; 799 800 man = ttm_manager_type(&rdev->mman.bdev, TTM_PL_VRAM); 801 /* this just adjusts TTM size idea, which sets lpfn to the correct value */ 802 man->size = size >> PAGE_SHIFT; 803 } 804 805 static vm_fault_t radeon_ttm_fault(struct vm_fault *vmf) 806 { 807 struct ttm_buffer_object *bo = vmf->vma->vm_private_data; 808 struct radeon_device *rdev = radeon_get_rdev(bo->bdev); 809 vm_fault_t ret; 810 811 down_read(&rdev->pm.mclk_lock); 812 813 ret = ttm_bo_vm_reserve(bo, vmf); 814 if (ret) 815 goto unlock_mclk; 816 817 ret = radeon_bo_fault_reserve_notify(bo); 818 if (ret) 819 goto unlock_resv; 820 821 ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, 822 TTM_BO_VM_NUM_PREFAULT, 1); 823 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) 824 goto unlock_mclk; 825 826 unlock_resv: 827 dma_resv_unlock(bo->base.resv); 828 829 unlock_mclk: 830 up_read(&rdev->pm.mclk_lock); 831 return ret; 832 } 833 834 static const struct vm_operations_struct radeon_ttm_vm_ops = { 835 .fault = radeon_ttm_fault, 836 .open = ttm_bo_vm_open, 837 .close = ttm_bo_vm_close, 838 .access = ttm_bo_vm_access 839 }; 840 841 int radeon_mmap(struct file *filp, struct vm_area_struct *vma) 842 { 843 int r; 844 struct drm_file *file_priv = filp->private_data; 845 struct radeon_device *rdev = file_priv->minor->dev->dev_private; 846 847 if (rdev == NULL) 848 return -EINVAL; 849 850 r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev); 851 if (unlikely(r != 0)) 852 return r; 853 854 vma->vm_ops = &radeon_ttm_vm_ops; 855 return 0; 856 } 857 858 #if defined(CONFIG_DEBUG_FS) 859 860 static int radeon_mm_vram_dump_table_show(struct seq_file *m, void *unused) 861 { 862 struct radeon_device *rdev = (struct radeon_device *)m->private; 863 struct ttm_resource_manager *man = ttm_manager_type(&rdev->mman.bdev, 864 TTM_PL_VRAM); 865 struct drm_printer p = drm_seq_file_printer(m); 866 867 man->func->debug(man, &p); 868 return 0; 869 } 870 871 static int radeon_ttm_page_pool_show(struct seq_file *m, void *data) 872 { 873 struct radeon_device *rdev = (struct radeon_device *)m->private; 874 875 return ttm_pool_debugfs(&rdev->mman.bdev.pool, m); 876 } 877 878 static int radeon_mm_gtt_dump_table_show(struct seq_file *m, void *unused) 879 { 880 struct radeon_device *rdev = (struct radeon_device *)m->private; 881 struct ttm_resource_manager *man = ttm_manager_type(&rdev->mman.bdev, 882 TTM_PL_TT); 883 struct drm_printer p = drm_seq_file_printer(m); 884 885 man->func->debug(man, &p); 886 return 0; 887 } 888 889 DEFINE_SHOW_ATTRIBUTE(radeon_mm_vram_dump_table); 890 DEFINE_SHOW_ATTRIBUTE(radeon_mm_gtt_dump_table); 891 DEFINE_SHOW_ATTRIBUTE(radeon_ttm_page_pool); 892 893 static int radeon_ttm_vram_open(struct inode *inode, struct file *filep) 894 { 895 struct radeon_device *rdev = inode->i_private; 896 i_size_write(inode, rdev->mc.mc_vram_size); 897 filep->private_data = inode->i_private; 898 return 0; 899 } 900 901 static ssize_t radeon_ttm_vram_read(struct file *f, char __user *buf, 902 size_t size, loff_t *pos) 903 { 904 struct radeon_device *rdev = f->private_data; 905 ssize_t result = 0; 906 int r; 907 908 if (size & 0x3 || *pos & 0x3) 909 return -EINVAL; 910 911 while (size) { 912 unsigned long flags; 913 uint32_t value; 914 915 if (*pos >= rdev->mc.mc_vram_size) 916 return result; 917 918 spin_lock_irqsave(&rdev->mmio_idx_lock, flags); 919 WREG32(RADEON_MM_INDEX, ((uint32_t)*pos) | 0x80000000); 920 if (rdev->family >= CHIP_CEDAR) 921 WREG32(EVERGREEN_MM_INDEX_HI, *pos >> 31); 922 value = RREG32(RADEON_MM_DATA); 923 spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); 924 925 r = put_user(value, (uint32_t __user *)buf); 926 if (r) 927 return r; 928 929 result += 4; 930 buf += 4; 931 *pos += 4; 932 size -= 4; 933 } 934 935 return result; 936 } 937 938 static const struct file_operations radeon_ttm_vram_fops = { 939 .owner = THIS_MODULE, 940 .open = radeon_ttm_vram_open, 941 .read = radeon_ttm_vram_read, 942 .llseek = default_llseek 943 }; 944 945 static int radeon_ttm_gtt_open(struct inode *inode, struct file *filep) 946 { 947 struct radeon_device *rdev = inode->i_private; 948 i_size_write(inode, rdev->mc.gtt_size); 949 filep->private_data = inode->i_private; 950 return 0; 951 } 952 953 static ssize_t radeon_ttm_gtt_read(struct file *f, char __user *buf, 954 size_t size, loff_t *pos) 955 { 956 struct radeon_device *rdev = f->private_data; 957 ssize_t result = 0; 958 int r; 959 960 while (size) { 961 loff_t p = *pos / PAGE_SIZE; 962 unsigned off = *pos & ~PAGE_MASK; 963 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); 964 struct page *page; 965 void *ptr; 966 967 if (p >= rdev->gart.num_cpu_pages) 968 return result; 969 970 page = rdev->gart.pages[p]; 971 if (page) { 972 ptr = kmap(page); 973 ptr += off; 974 975 r = copy_to_user(buf, ptr, cur_size); 976 kunmap(rdev->gart.pages[p]); 977 } else 978 r = clear_user(buf, cur_size); 979 980 if (r) 981 return -EFAULT; 982 983 result += cur_size; 984 buf += cur_size; 985 *pos += cur_size; 986 size -= cur_size; 987 } 988 989 return result; 990 } 991 992 static const struct file_operations radeon_ttm_gtt_fops = { 993 .owner = THIS_MODULE, 994 .open = radeon_ttm_gtt_open, 995 .read = radeon_ttm_gtt_read, 996 .llseek = default_llseek 997 }; 998 999 #endif 1000 1001 static void radeon_ttm_debugfs_init(struct radeon_device *rdev) 1002 { 1003 #if defined(CONFIG_DEBUG_FS) 1004 struct drm_minor *minor = rdev->ddev->primary; 1005 struct dentry *root = minor->debugfs_root; 1006 1007 debugfs_create_file("radeon_vram", 0444, root, rdev, 1008 &radeon_ttm_vram_fops); 1009 1010 debugfs_create_file("radeon_gtt", 0444, root, rdev, 1011 &radeon_ttm_gtt_fops); 1012 1013 debugfs_create_file("radeon_vram_mm", 0444, root, rdev, 1014 &radeon_mm_vram_dump_table_fops); 1015 debugfs_create_file("radeon_gtt_mm", 0444, root, rdev, 1016 &radeon_mm_gtt_dump_table_fops); 1017 debugfs_create_file("ttm_page_pool", 0444, root, rdev, 1018 &radeon_ttm_page_pool_fops); 1019 #endif 1020 } 1021