1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 33 #include <linux/dma-mapping.h> 34 #include <linux/pagemap.h> 35 #include <linux/pci.h> 36 #include <linux/seq_file.h> 37 #include <linux/slab.h> 38 #include <linux/swap.h> 39 #include <linux/swiotlb.h> 40 41 #include <drm/drm_agpsupport.h> 42 #include <drm/drm_debugfs.h> 43 #include <drm/drm_device.h> 44 #include <drm/drm_file.h> 45 #include <drm/drm_prime.h> 46 #include <drm/radeon_drm.h> 47 #include <drm/ttm/ttm_bo_api.h> 48 #include <drm/ttm/ttm_bo_driver.h> 49 #include <drm/ttm/ttm_module.h> 50 #include <drm/ttm/ttm_placement.h> 51 52 #include "radeon_reg.h" 53 #include "radeon.h" 54 #include "radeon_ttm.h" 55 56 static int radeon_ttm_debugfs_init(struct radeon_device *rdev); 57 static void radeon_ttm_debugfs_fini(struct radeon_device *rdev); 58 59 static int radeon_ttm_tt_bind(struct ttm_bo_device *bdev, 60 struct ttm_tt *ttm, 61 struct ttm_resource *bo_mem); 62 static void radeon_ttm_tt_unbind(struct ttm_bo_device *bdev, 63 struct ttm_tt *ttm); 64 65 struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev) 66 { 67 struct radeon_mman *mman; 68 struct radeon_device *rdev; 69 70 mman = container_of(bdev, struct radeon_mman, bdev); 71 rdev = container_of(mman, struct radeon_device, mman); 72 return rdev; 73 } 74 75 static int radeon_ttm_init_vram(struct radeon_device *rdev) 76 { 77 return ttm_range_man_init(&rdev->mman.bdev, TTM_PL_VRAM, 78 false, rdev->mc.real_vram_size >> PAGE_SHIFT); 79 } 80 81 static int radeon_ttm_init_gtt(struct radeon_device *rdev) 82 { 83 return ttm_range_man_init(&rdev->mman.bdev, TTM_PL_TT, 84 true, rdev->mc.gtt_size >> PAGE_SHIFT); 85 } 86 87 static void radeon_evict_flags(struct ttm_buffer_object *bo, 88 struct ttm_placement *placement) 89 { 90 static const struct ttm_place placements = { 91 .fpfn = 0, 92 .lpfn = 0, 93 .mem_type = TTM_PL_SYSTEM, 94 .flags = 0 95 }; 96 97 struct radeon_bo *rbo; 98 99 if (!radeon_ttm_bo_is_radeon_bo(bo)) { 100 placement->placement = &placements; 101 placement->busy_placement = &placements; 102 placement->num_placement = 1; 103 placement->num_busy_placement = 1; 104 return; 105 } 106 rbo = container_of(bo, struct radeon_bo, tbo); 107 switch (bo->mem.mem_type) { 108 case TTM_PL_VRAM: 109 if (rbo->rdev->ring[radeon_copy_ring_index(rbo->rdev)].ready == false) 110 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 111 else if (rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size && 112 bo->mem.start < (rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT)) { 113 unsigned fpfn = rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; 114 int i; 115 116 /* Try evicting to the CPU inaccessible part of VRAM 117 * first, but only set GTT as busy placement, so this 118 * BO will be evicted to GTT rather than causing other 119 * BOs to be evicted from VRAM 120 */ 121 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM | 122 RADEON_GEM_DOMAIN_GTT); 123 rbo->placement.num_busy_placement = 0; 124 for (i = 0; i < rbo->placement.num_placement; i++) { 125 if (rbo->placements[i].mem_type == TTM_PL_VRAM) { 126 if (rbo->placements[i].fpfn < fpfn) 127 rbo->placements[i].fpfn = fpfn; 128 } else { 129 rbo->placement.busy_placement = 130 &rbo->placements[i]; 131 rbo->placement.num_busy_placement = 1; 132 } 133 } 134 } else 135 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); 136 break; 137 case TTM_PL_TT: 138 default: 139 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 140 } 141 *placement = rbo->placement; 142 } 143 144 static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp) 145 { 146 struct radeon_bo *rbo = container_of(bo, struct radeon_bo, tbo); 147 struct radeon_device *rdev = radeon_get_rdev(bo->bdev); 148 149 if (radeon_ttm_tt_has_userptr(rdev, bo->ttm)) 150 return -EPERM; 151 return drm_vma_node_verify_access(&rbo->tbo.base.vma_node, 152 filp->private_data); 153 } 154 155 static int radeon_move_blit(struct ttm_buffer_object *bo, 156 bool evict, 157 struct ttm_resource *new_mem, 158 struct ttm_resource *old_mem) 159 { 160 struct radeon_device *rdev; 161 uint64_t old_start, new_start; 162 struct radeon_fence *fence; 163 unsigned num_pages; 164 int r, ridx; 165 166 rdev = radeon_get_rdev(bo->bdev); 167 ridx = radeon_copy_ring_index(rdev); 168 old_start = (u64)old_mem->start << PAGE_SHIFT; 169 new_start = (u64)new_mem->start << PAGE_SHIFT; 170 171 switch (old_mem->mem_type) { 172 case TTM_PL_VRAM: 173 old_start += rdev->mc.vram_start; 174 break; 175 case TTM_PL_TT: 176 old_start += rdev->mc.gtt_start; 177 break; 178 default: 179 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 180 return -EINVAL; 181 } 182 switch (new_mem->mem_type) { 183 case TTM_PL_VRAM: 184 new_start += rdev->mc.vram_start; 185 break; 186 case TTM_PL_TT: 187 new_start += rdev->mc.gtt_start; 188 break; 189 default: 190 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 191 return -EINVAL; 192 } 193 if (!rdev->ring[ridx].ready) { 194 DRM_ERROR("Trying to move memory with ring turned off.\n"); 195 return -EINVAL; 196 } 197 198 BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); 199 200 num_pages = new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 201 fence = radeon_copy(rdev, old_start, new_start, num_pages, bo->base.resv); 202 if (IS_ERR(fence)) 203 return PTR_ERR(fence); 204 205 r = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, false, new_mem); 206 radeon_fence_unref(&fence); 207 return r; 208 } 209 210 static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, 211 struct ttm_operation_ctx *ctx, 212 struct ttm_resource *new_mem, 213 struct ttm_place *hop) 214 { 215 struct radeon_device *rdev; 216 struct radeon_bo *rbo; 217 struct ttm_resource *old_mem = &bo->mem; 218 int r; 219 220 if ((old_mem->mem_type == TTM_PL_SYSTEM && 221 new_mem->mem_type == TTM_PL_VRAM) || 222 (old_mem->mem_type == TTM_PL_VRAM && 223 new_mem->mem_type == TTM_PL_SYSTEM)) { 224 hop->fpfn = 0; 225 hop->lpfn = 0; 226 hop->mem_type = TTM_PL_TT; 227 hop->flags = 0; 228 return -EMULTIHOP; 229 } 230 231 if (new_mem->mem_type == TTM_PL_TT) { 232 r = radeon_ttm_tt_bind(bo->bdev, bo->ttm, new_mem); 233 if (r) 234 return r; 235 } 236 radeon_bo_move_notify(bo, evict, new_mem); 237 238 r = ttm_bo_wait_ctx(bo, ctx); 239 if (r) 240 goto fail; 241 242 /* Can't move a pinned BO */ 243 rbo = container_of(bo, struct radeon_bo, tbo); 244 if (WARN_ON_ONCE(rbo->tbo.pin_count > 0)) 245 return -EINVAL; 246 247 rdev = radeon_get_rdev(bo->bdev); 248 if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 249 ttm_bo_move_null(bo, new_mem); 250 return 0; 251 } 252 if (old_mem->mem_type == TTM_PL_SYSTEM && 253 new_mem->mem_type == TTM_PL_TT) { 254 ttm_bo_move_null(bo, new_mem); 255 return 0; 256 } 257 258 if (old_mem->mem_type == TTM_PL_TT && 259 new_mem->mem_type == TTM_PL_SYSTEM) { 260 radeon_ttm_tt_unbind(bo->bdev, bo->ttm); 261 ttm_resource_free(bo, &bo->mem); 262 ttm_bo_assign_mem(bo, new_mem); 263 return 0; 264 } 265 if (!rdev->ring[radeon_copy_ring_index(rdev)].ready || 266 rdev->asic->copy.copy == NULL) { 267 /* use memcpy */ 268 goto memcpy; 269 } 270 271 r = radeon_move_blit(bo, evict, new_mem, old_mem); 272 if (r) { 273 memcpy: 274 r = ttm_bo_move_memcpy(bo, ctx, new_mem); 275 if (r) { 276 goto fail; 277 } 278 } 279 280 /* update statistics */ 281 atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &rdev->num_bytes_moved); 282 return 0; 283 fail: 284 swap(*new_mem, bo->mem); 285 radeon_bo_move_notify(bo, false, new_mem); 286 swap(*new_mem, bo->mem); 287 return r; 288 } 289 290 static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_resource *mem) 291 { 292 struct radeon_device *rdev = radeon_get_rdev(bdev); 293 size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT; 294 295 switch (mem->mem_type) { 296 case TTM_PL_SYSTEM: 297 /* system memory */ 298 return 0; 299 case TTM_PL_TT: 300 #if IS_ENABLED(CONFIG_AGP) 301 if (rdev->flags & RADEON_IS_AGP) { 302 /* RADEON_IS_AGP is set only if AGP is active */ 303 mem->bus.offset = (mem->start << PAGE_SHIFT) + 304 rdev->mc.agp_base; 305 mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture; 306 mem->bus.caching = ttm_write_combined; 307 } 308 #endif 309 break; 310 case TTM_PL_VRAM: 311 mem->bus.offset = mem->start << PAGE_SHIFT; 312 /* check if it's visible */ 313 if ((mem->bus.offset + bus_size) > rdev->mc.visible_vram_size) 314 return -EINVAL; 315 mem->bus.offset += rdev->mc.aper_base; 316 mem->bus.is_iomem = true; 317 mem->bus.caching = ttm_write_combined; 318 #ifdef __alpha__ 319 /* 320 * Alpha: use bus.addr to hold the ioremap() return, 321 * so we can modify bus.base below. 322 */ 323 mem->bus.addr = ioremap_wc(mem->bus.offset, bus_size); 324 if (!mem->bus.addr) 325 return -ENOMEM; 326 327 /* 328 * Alpha: Use just the bus offset plus 329 * the hose/domain memory base for bus.base. 330 * It then can be used to build PTEs for VRAM 331 * access, as done in ttm_bo_vm_fault(). 332 */ 333 mem->bus.offset = (mem->bus.offset & 0x0ffffffffUL) + 334 rdev->ddev->hose->dense_mem_base; 335 #endif 336 break; 337 default: 338 return -EINVAL; 339 } 340 return 0; 341 } 342 343 /* 344 * TTM backend functions. 345 */ 346 struct radeon_ttm_tt { 347 struct ttm_tt ttm; 348 u64 offset; 349 350 uint64_t userptr; 351 struct mm_struct *usermm; 352 uint32_t userflags; 353 bool bound; 354 }; 355 356 /* prepare the sg table with the user pages */ 357 static int radeon_ttm_tt_pin_userptr(struct ttm_bo_device *bdev, struct ttm_tt *ttm) 358 { 359 struct radeon_device *rdev = radeon_get_rdev(bdev); 360 struct radeon_ttm_tt *gtt = (void *)ttm; 361 unsigned pinned = 0; 362 int r; 363 364 int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 365 enum dma_data_direction direction = write ? 366 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 367 368 if (current->mm != gtt->usermm) 369 return -EPERM; 370 371 if (gtt->userflags & RADEON_GEM_USERPTR_ANONONLY) { 372 /* check that we only pin down anonymous memory 373 to prevent problems with writeback */ 374 unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE; 375 struct vm_area_struct *vma; 376 vma = find_vma(gtt->usermm, gtt->userptr); 377 if (!vma || vma->vm_file || vma->vm_end < end) 378 return -EPERM; 379 } 380 381 do { 382 unsigned num_pages = ttm->num_pages - pinned; 383 uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE; 384 struct page **pages = ttm->pages + pinned; 385 386 r = get_user_pages(userptr, num_pages, write ? FOLL_WRITE : 0, 387 pages, NULL); 388 if (r < 0) 389 goto release_pages; 390 391 pinned += r; 392 393 } while (pinned < ttm->num_pages); 394 395 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, 396 ttm->num_pages << PAGE_SHIFT, 397 GFP_KERNEL); 398 if (r) 399 goto release_sg; 400 401 r = dma_map_sgtable(rdev->dev, ttm->sg, direction, 0); 402 if (r) 403 goto release_sg; 404 405 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 406 gtt->ttm.dma_address, ttm->num_pages); 407 408 return 0; 409 410 release_sg: 411 kfree(ttm->sg); 412 413 release_pages: 414 release_pages(ttm->pages, pinned); 415 return r; 416 } 417 418 static void radeon_ttm_tt_unpin_userptr(struct ttm_bo_device *bdev, struct ttm_tt *ttm) 419 { 420 struct radeon_device *rdev = radeon_get_rdev(bdev); 421 struct radeon_ttm_tt *gtt = (void *)ttm; 422 struct sg_page_iter sg_iter; 423 424 int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 425 enum dma_data_direction direction = write ? 426 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 427 428 /* double check that we don't free the table twice */ 429 if (!ttm->sg->sgl) 430 return; 431 432 /* free the sg table and pages again */ 433 dma_unmap_sgtable(rdev->dev, ttm->sg, direction, 0); 434 435 for_each_sgtable_page(ttm->sg, &sg_iter, 0) { 436 struct page *page = sg_page_iter_page(&sg_iter); 437 if (!(gtt->userflags & RADEON_GEM_USERPTR_READONLY)) 438 set_page_dirty(page); 439 440 mark_page_accessed(page); 441 put_page(page); 442 } 443 444 sg_free_table(ttm->sg); 445 } 446 447 static bool radeon_ttm_backend_is_bound(struct ttm_tt *ttm) 448 { 449 struct radeon_ttm_tt *gtt = (void*)ttm; 450 451 return (gtt->bound); 452 } 453 454 static int radeon_ttm_backend_bind(struct ttm_bo_device *bdev, 455 struct ttm_tt *ttm, 456 struct ttm_resource *bo_mem) 457 { 458 struct radeon_ttm_tt *gtt = (void*)ttm; 459 struct radeon_device *rdev = radeon_get_rdev(bdev); 460 uint32_t flags = RADEON_GART_PAGE_VALID | RADEON_GART_PAGE_READ | 461 RADEON_GART_PAGE_WRITE; 462 int r; 463 464 if (gtt->bound) 465 return 0; 466 467 if (gtt->userptr) { 468 radeon_ttm_tt_pin_userptr(bdev, ttm); 469 flags &= ~RADEON_GART_PAGE_WRITE; 470 } 471 472 gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); 473 if (!ttm->num_pages) { 474 WARN(1, "nothing to bind %u pages for mreg %p back %p!\n", 475 ttm->num_pages, bo_mem, ttm); 476 } 477 if (ttm->caching == ttm_cached) 478 flags |= RADEON_GART_PAGE_SNOOP; 479 r = radeon_gart_bind(rdev, gtt->offset, ttm->num_pages, 480 ttm->pages, gtt->ttm.dma_address, flags); 481 if (r) { 482 DRM_ERROR("failed to bind %u pages at 0x%08X\n", 483 ttm->num_pages, (unsigned)gtt->offset); 484 return r; 485 } 486 gtt->bound = true; 487 return 0; 488 } 489 490 static void radeon_ttm_backend_unbind(struct ttm_bo_device *bdev, struct ttm_tt *ttm) 491 { 492 struct radeon_ttm_tt *gtt = (void *)ttm; 493 struct radeon_device *rdev = radeon_get_rdev(bdev); 494 495 if (!gtt->bound) 496 return; 497 498 radeon_gart_unbind(rdev, gtt->offset, ttm->num_pages); 499 500 if (gtt->userptr) 501 radeon_ttm_tt_unpin_userptr(bdev, ttm); 502 gtt->bound = false; 503 } 504 505 static void radeon_ttm_backend_destroy(struct ttm_bo_device *bdev, struct ttm_tt *ttm) 506 { 507 struct radeon_ttm_tt *gtt = (void *)ttm; 508 509 radeon_ttm_backend_unbind(bdev, ttm); 510 ttm_tt_destroy_common(bdev, ttm); 511 512 ttm_tt_fini(>t->ttm); 513 kfree(gtt); 514 } 515 516 static struct ttm_tt *radeon_ttm_tt_create(struct ttm_buffer_object *bo, 517 uint32_t page_flags) 518 { 519 struct radeon_ttm_tt *gtt; 520 enum ttm_caching caching; 521 struct radeon_bo *rbo; 522 #if IS_ENABLED(CONFIG_AGP) 523 struct radeon_device *rdev = radeon_get_rdev(bo->bdev); 524 525 if (rdev->flags & RADEON_IS_AGP) { 526 return ttm_agp_tt_create(bo, rdev->ddev->agp->bridge, 527 page_flags); 528 } 529 #endif 530 rbo = container_of(bo, struct radeon_bo, tbo); 531 532 gtt = kzalloc(sizeof(struct radeon_ttm_tt), GFP_KERNEL); 533 if (gtt == NULL) { 534 return NULL; 535 } 536 537 if (rbo->flags & RADEON_GEM_GTT_UC) 538 caching = ttm_uncached; 539 else if (rbo->flags & RADEON_GEM_GTT_WC) 540 caching = ttm_write_combined; 541 else 542 caching = ttm_cached; 543 544 if (ttm_dma_tt_init(>t->ttm, bo, page_flags, caching)) { 545 kfree(gtt); 546 return NULL; 547 } 548 return >t->ttm; 549 } 550 551 static struct radeon_ttm_tt *radeon_ttm_tt_to_gtt(struct radeon_device *rdev, 552 struct ttm_tt *ttm) 553 { 554 #if IS_ENABLED(CONFIG_AGP) 555 if (rdev->flags & RADEON_IS_AGP) 556 return NULL; 557 #endif 558 559 if (!ttm) 560 return NULL; 561 return container_of(ttm, struct radeon_ttm_tt, ttm); 562 } 563 564 static int radeon_ttm_tt_populate(struct ttm_bo_device *bdev, 565 struct ttm_tt *ttm, 566 struct ttm_operation_ctx *ctx) 567 { 568 struct radeon_device *rdev = radeon_get_rdev(bdev); 569 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); 570 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 571 572 if (gtt && gtt->userptr) { 573 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); 574 if (!ttm->sg) 575 return -ENOMEM; 576 577 ttm->page_flags |= TTM_PAGE_FLAG_SG; 578 return 0; 579 } 580 581 if (slave && ttm->sg) { 582 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 583 gtt->ttm.dma_address, ttm->num_pages); 584 return 0; 585 } 586 587 return ttm_pool_alloc(&rdev->mman.bdev.pool, ttm, ctx); 588 } 589 590 static void radeon_ttm_tt_unpopulate(struct ttm_bo_device *bdev, struct ttm_tt *ttm) 591 { 592 struct radeon_device *rdev = radeon_get_rdev(bdev); 593 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); 594 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 595 596 if (gtt && gtt->userptr) { 597 kfree(ttm->sg); 598 ttm->page_flags &= ~TTM_PAGE_FLAG_SG; 599 return; 600 } 601 602 if (slave) 603 return; 604 605 return ttm_pool_free(&rdev->mman.bdev.pool, ttm); 606 } 607 608 int radeon_ttm_tt_set_userptr(struct radeon_device *rdev, 609 struct ttm_tt *ttm, uint64_t addr, 610 uint32_t flags) 611 { 612 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); 613 614 if (gtt == NULL) 615 return -EINVAL; 616 617 gtt->userptr = addr; 618 gtt->usermm = current->mm; 619 gtt->userflags = flags; 620 return 0; 621 } 622 623 bool radeon_ttm_tt_is_bound(struct ttm_bo_device *bdev, 624 struct ttm_tt *ttm) 625 { 626 #if IS_ENABLED(CONFIG_AGP) 627 struct radeon_device *rdev = radeon_get_rdev(bdev); 628 if (rdev->flags & RADEON_IS_AGP) 629 return ttm_agp_is_bound(ttm); 630 #endif 631 return radeon_ttm_backend_is_bound(ttm); 632 } 633 634 static int radeon_ttm_tt_bind(struct ttm_bo_device *bdev, 635 struct ttm_tt *ttm, 636 struct ttm_resource *bo_mem) 637 { 638 #if IS_ENABLED(CONFIG_AGP) 639 struct radeon_device *rdev = radeon_get_rdev(bdev); 640 #endif 641 642 if (!bo_mem) 643 return -EINVAL; 644 #if IS_ENABLED(CONFIG_AGP) 645 if (rdev->flags & RADEON_IS_AGP) 646 return ttm_agp_bind(ttm, bo_mem); 647 #endif 648 649 return radeon_ttm_backend_bind(bdev, ttm, bo_mem); 650 } 651 652 static void radeon_ttm_tt_unbind(struct ttm_bo_device *bdev, 653 struct ttm_tt *ttm) 654 { 655 #if IS_ENABLED(CONFIG_AGP) 656 struct radeon_device *rdev = radeon_get_rdev(bdev); 657 658 if (rdev->flags & RADEON_IS_AGP) { 659 ttm_agp_unbind(ttm); 660 return; 661 } 662 #endif 663 radeon_ttm_backend_unbind(bdev, ttm); 664 } 665 666 static void radeon_ttm_tt_destroy(struct ttm_bo_device *bdev, 667 struct ttm_tt *ttm) 668 { 669 #if IS_ENABLED(CONFIG_AGP) 670 struct radeon_device *rdev = radeon_get_rdev(bdev); 671 672 if (rdev->flags & RADEON_IS_AGP) { 673 ttm_agp_unbind(ttm); 674 ttm_tt_destroy_common(bdev, ttm); 675 ttm_agp_destroy(ttm); 676 return; 677 } 678 #endif 679 radeon_ttm_backend_destroy(bdev, ttm); 680 } 681 682 bool radeon_ttm_tt_has_userptr(struct radeon_device *rdev, 683 struct ttm_tt *ttm) 684 { 685 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); 686 687 if (gtt == NULL) 688 return false; 689 690 return !!gtt->userptr; 691 } 692 693 bool radeon_ttm_tt_is_readonly(struct radeon_device *rdev, 694 struct ttm_tt *ttm) 695 { 696 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(rdev, ttm); 697 698 if (gtt == NULL) 699 return false; 700 701 return !!(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 702 } 703 704 static void 705 radeon_bo_delete_mem_notify(struct ttm_buffer_object *bo) 706 { 707 radeon_bo_move_notify(bo, false, NULL); 708 } 709 710 static struct ttm_bo_driver radeon_bo_driver = { 711 .ttm_tt_create = &radeon_ttm_tt_create, 712 .ttm_tt_populate = &radeon_ttm_tt_populate, 713 .ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate, 714 .ttm_tt_destroy = &radeon_ttm_tt_destroy, 715 .eviction_valuable = ttm_bo_eviction_valuable, 716 .evict_flags = &radeon_evict_flags, 717 .move = &radeon_bo_move, 718 .verify_access = &radeon_verify_access, 719 .delete_mem_notify = &radeon_bo_delete_mem_notify, 720 .io_mem_reserve = &radeon_ttm_io_mem_reserve, 721 }; 722 723 int radeon_ttm_init(struct radeon_device *rdev) 724 { 725 int r; 726 727 /* No others user of address space so set it to 0 */ 728 r = ttm_bo_device_init(&rdev->mman.bdev, &radeon_bo_driver, rdev->dev, 729 rdev->ddev->anon_inode->i_mapping, 730 rdev->ddev->vma_offset_manager, 731 rdev->need_swiotlb, 732 dma_addressing_limited(&rdev->pdev->dev)); 733 if (r) { 734 DRM_ERROR("failed initializing buffer object driver(%d).\n", r); 735 return r; 736 } 737 rdev->mman.initialized = true; 738 739 ttm_pool_init(&rdev->mman.bdev.pool, rdev->dev, rdev->need_swiotlb, 740 dma_addressing_limited(&rdev->pdev->dev)); 741 742 r = radeon_ttm_init_vram(rdev); 743 if (r) { 744 DRM_ERROR("Failed initializing VRAM heap.\n"); 745 return r; 746 } 747 /* Change the size here instead of the init above so only lpfn is affected */ 748 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 749 750 r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true, 751 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 752 NULL, &rdev->stolen_vga_memory); 753 if (r) { 754 return r; 755 } 756 r = radeon_bo_reserve(rdev->stolen_vga_memory, false); 757 if (r) 758 return r; 759 r = radeon_bo_pin(rdev->stolen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL); 760 radeon_bo_unreserve(rdev->stolen_vga_memory); 761 if (r) { 762 radeon_bo_unref(&rdev->stolen_vga_memory); 763 return r; 764 } 765 DRM_INFO("radeon: %uM of VRAM memory ready\n", 766 (unsigned) (rdev->mc.real_vram_size / (1024 * 1024))); 767 768 r = radeon_ttm_init_gtt(rdev); 769 if (r) { 770 DRM_ERROR("Failed initializing GTT heap.\n"); 771 return r; 772 } 773 DRM_INFO("radeon: %uM of GTT memory ready.\n", 774 (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); 775 776 r = radeon_ttm_debugfs_init(rdev); 777 if (r) { 778 DRM_ERROR("Failed to init debugfs\n"); 779 return r; 780 } 781 return 0; 782 } 783 784 void radeon_ttm_fini(struct radeon_device *rdev) 785 { 786 int r; 787 788 if (!rdev->mman.initialized) 789 return; 790 radeon_ttm_debugfs_fini(rdev); 791 if (rdev->stolen_vga_memory) { 792 r = radeon_bo_reserve(rdev->stolen_vga_memory, false); 793 if (r == 0) { 794 radeon_bo_unpin(rdev->stolen_vga_memory); 795 radeon_bo_unreserve(rdev->stolen_vga_memory); 796 } 797 radeon_bo_unref(&rdev->stolen_vga_memory); 798 } 799 ttm_range_man_fini(&rdev->mman.bdev, TTM_PL_VRAM); 800 ttm_range_man_fini(&rdev->mman.bdev, TTM_PL_TT); 801 ttm_bo_device_release(&rdev->mman.bdev); 802 radeon_gart_fini(rdev); 803 rdev->mman.initialized = false; 804 DRM_INFO("radeon: ttm finalized\n"); 805 } 806 807 /* this should only be called at bootup or when userspace 808 * isn't running */ 809 void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size) 810 { 811 struct ttm_resource_manager *man; 812 813 if (!rdev->mman.initialized) 814 return; 815 816 man = ttm_manager_type(&rdev->mman.bdev, TTM_PL_VRAM); 817 /* this just adjusts TTM size idea, which sets lpfn to the correct value */ 818 man->size = size >> PAGE_SHIFT; 819 } 820 821 static vm_fault_t radeon_ttm_fault(struct vm_fault *vmf) 822 { 823 struct ttm_buffer_object *bo = vmf->vma->vm_private_data; 824 struct radeon_device *rdev = radeon_get_rdev(bo->bdev); 825 vm_fault_t ret; 826 827 down_read(&rdev->pm.mclk_lock); 828 829 ret = ttm_bo_vm_reserve(bo, vmf); 830 if (ret) 831 goto unlock_mclk; 832 833 ret = radeon_bo_fault_reserve_notify(bo); 834 if (ret) 835 goto unlock_resv; 836 837 ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, 838 TTM_BO_VM_NUM_PREFAULT, 1); 839 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) 840 goto unlock_mclk; 841 842 unlock_resv: 843 dma_resv_unlock(bo->base.resv); 844 845 unlock_mclk: 846 up_read(&rdev->pm.mclk_lock); 847 return ret; 848 } 849 850 static struct vm_operations_struct radeon_ttm_vm_ops = { 851 .fault = radeon_ttm_fault, 852 .open = ttm_bo_vm_open, 853 .close = ttm_bo_vm_close, 854 .access = ttm_bo_vm_access 855 }; 856 857 int radeon_mmap(struct file *filp, struct vm_area_struct *vma) 858 { 859 int r; 860 struct drm_file *file_priv = filp->private_data; 861 struct radeon_device *rdev = file_priv->minor->dev->dev_private; 862 863 if (rdev == NULL) 864 return -EINVAL; 865 866 r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev); 867 if (unlikely(r != 0)) 868 return r; 869 870 vma->vm_ops = &radeon_ttm_vm_ops; 871 return 0; 872 } 873 874 #if defined(CONFIG_DEBUG_FS) 875 876 static int radeon_mm_dump_table(struct seq_file *m, void *data) 877 { 878 struct drm_info_node *node = (struct drm_info_node *)m->private; 879 unsigned ttm_pl = *(int*)node->info_ent->data; 880 struct drm_device *dev = node->minor->dev; 881 struct radeon_device *rdev = dev->dev_private; 882 struct ttm_resource_manager *man = ttm_manager_type(&rdev->mman.bdev, ttm_pl); 883 struct drm_printer p = drm_seq_file_printer(m); 884 885 man->func->debug(man, &p); 886 return 0; 887 } 888 889 static int radeon_ttm_pool_debugfs(struct seq_file *m, void *data) 890 { 891 struct drm_info_node *node = (struct drm_info_node *)m->private; 892 struct drm_device *dev = node->minor->dev; 893 struct radeon_device *rdev = dev->dev_private; 894 895 return ttm_pool_debugfs(&rdev->mman.bdev.pool, m); 896 } 897 898 static int ttm_pl_vram = TTM_PL_VRAM; 899 static int ttm_pl_tt = TTM_PL_TT; 900 901 static struct drm_info_list radeon_ttm_debugfs_list[] = { 902 {"radeon_vram_mm", radeon_mm_dump_table, 0, &ttm_pl_vram}, 903 {"radeon_gtt_mm", radeon_mm_dump_table, 0, &ttm_pl_tt}, 904 {"ttm_page_pool", radeon_ttm_pool_debugfs, 0, NULL} 905 }; 906 907 static int radeon_ttm_vram_open(struct inode *inode, struct file *filep) 908 { 909 struct radeon_device *rdev = inode->i_private; 910 i_size_write(inode, rdev->mc.mc_vram_size); 911 filep->private_data = inode->i_private; 912 return 0; 913 } 914 915 static ssize_t radeon_ttm_vram_read(struct file *f, char __user *buf, 916 size_t size, loff_t *pos) 917 { 918 struct radeon_device *rdev = f->private_data; 919 ssize_t result = 0; 920 int r; 921 922 if (size & 0x3 || *pos & 0x3) 923 return -EINVAL; 924 925 while (size) { 926 unsigned long flags; 927 uint32_t value; 928 929 if (*pos >= rdev->mc.mc_vram_size) 930 return result; 931 932 spin_lock_irqsave(&rdev->mmio_idx_lock, flags); 933 WREG32(RADEON_MM_INDEX, ((uint32_t)*pos) | 0x80000000); 934 if (rdev->family >= CHIP_CEDAR) 935 WREG32(EVERGREEN_MM_INDEX_HI, *pos >> 31); 936 value = RREG32(RADEON_MM_DATA); 937 spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); 938 939 r = put_user(value, (uint32_t *)buf); 940 if (r) 941 return r; 942 943 result += 4; 944 buf += 4; 945 *pos += 4; 946 size -= 4; 947 } 948 949 return result; 950 } 951 952 static const struct file_operations radeon_ttm_vram_fops = { 953 .owner = THIS_MODULE, 954 .open = radeon_ttm_vram_open, 955 .read = radeon_ttm_vram_read, 956 .llseek = default_llseek 957 }; 958 959 static int radeon_ttm_gtt_open(struct inode *inode, struct file *filep) 960 { 961 struct radeon_device *rdev = inode->i_private; 962 i_size_write(inode, rdev->mc.gtt_size); 963 filep->private_data = inode->i_private; 964 return 0; 965 } 966 967 static ssize_t radeon_ttm_gtt_read(struct file *f, char __user *buf, 968 size_t size, loff_t *pos) 969 { 970 struct radeon_device *rdev = f->private_data; 971 ssize_t result = 0; 972 int r; 973 974 while (size) { 975 loff_t p = *pos / PAGE_SIZE; 976 unsigned off = *pos & ~PAGE_MASK; 977 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); 978 struct page *page; 979 void *ptr; 980 981 if (p >= rdev->gart.num_cpu_pages) 982 return result; 983 984 page = rdev->gart.pages[p]; 985 if (page) { 986 ptr = kmap(page); 987 ptr += off; 988 989 r = copy_to_user(buf, ptr, cur_size); 990 kunmap(rdev->gart.pages[p]); 991 } else 992 r = clear_user(buf, cur_size); 993 994 if (r) 995 return -EFAULT; 996 997 result += cur_size; 998 buf += cur_size; 999 *pos += cur_size; 1000 size -= cur_size; 1001 } 1002 1003 return result; 1004 } 1005 1006 static const struct file_operations radeon_ttm_gtt_fops = { 1007 .owner = THIS_MODULE, 1008 .open = radeon_ttm_gtt_open, 1009 .read = radeon_ttm_gtt_read, 1010 .llseek = default_llseek 1011 }; 1012 1013 #endif 1014 1015 static int radeon_ttm_debugfs_init(struct radeon_device *rdev) 1016 { 1017 #if defined(CONFIG_DEBUG_FS) 1018 unsigned count; 1019 1020 struct drm_minor *minor = rdev->ddev->primary; 1021 struct dentry *root = minor->debugfs_root; 1022 1023 rdev->mman.vram = debugfs_create_file("radeon_vram", S_IFREG | S_IRUGO, 1024 root, rdev, 1025 &radeon_ttm_vram_fops); 1026 1027 rdev->mman.gtt = debugfs_create_file("radeon_gtt", S_IFREG | S_IRUGO, 1028 root, rdev, &radeon_ttm_gtt_fops); 1029 1030 count = ARRAY_SIZE(radeon_ttm_debugfs_list); 1031 1032 return radeon_debugfs_add_files(rdev, radeon_ttm_debugfs_list, count); 1033 #else 1034 1035 return 0; 1036 #endif 1037 } 1038 1039 static void radeon_ttm_debugfs_fini(struct radeon_device *rdev) 1040 { 1041 #if defined(CONFIG_DEBUG_FS) 1042 1043 debugfs_remove(rdev->mman.vram); 1044 rdev->mman.vram = NULL; 1045 1046 debugfs_remove(rdev->mman.gtt); 1047 rdev->mman.gtt = NULL; 1048 #endif 1049 } 1050