1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 33 #include <linux/dma-mapping.h> 34 #include <linux/pagemap.h> 35 #include <linux/pci.h> 36 #include <linux/seq_file.h> 37 #include <linux/slab.h> 38 #include <linux/swap.h> 39 #include <linux/swiotlb.h> 40 41 #include <drm/drm_agpsupport.h> 42 #include <drm/drm_debugfs.h> 43 #include <drm/drm_device.h> 44 #include <drm/drm_file.h> 45 #include <drm/drm_prime.h> 46 #include <drm/radeon_drm.h> 47 #include <drm/ttm/ttm_bo_api.h> 48 #include <drm/ttm/ttm_bo_driver.h> 49 #include <drm/ttm/ttm_module.h> 50 #include <drm/ttm/ttm_page_alloc.h> 51 #include <drm/ttm/ttm_placement.h> 52 53 #include "radeon_reg.h" 54 #include "radeon.h" 55 56 static int radeon_ttm_debugfs_init(struct radeon_device *rdev); 57 static void radeon_ttm_debugfs_fini(struct radeon_device *rdev); 58 59 static struct radeon_device *radeon_get_rdev(struct ttm_bo_device *bdev) 60 { 61 struct radeon_mman *mman; 62 struct radeon_device *rdev; 63 64 mman = container_of(bdev, struct radeon_mman, bdev); 65 rdev = container_of(mman, struct radeon_device, mman); 66 return rdev; 67 } 68 69 static int radeon_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, 70 struct ttm_mem_type_manager *man) 71 { 72 struct radeon_device *rdev; 73 74 rdev = radeon_get_rdev(bdev); 75 76 switch (type) { 77 case TTM_PL_SYSTEM: 78 /* System memory */ 79 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 80 man->available_caching = TTM_PL_MASK_CACHING; 81 man->default_caching = TTM_PL_FLAG_CACHED; 82 break; 83 case TTM_PL_TT: 84 man->func = &ttm_bo_manager_func; 85 man->gpu_offset = rdev->mc.gtt_start; 86 man->available_caching = TTM_PL_MASK_CACHING; 87 man->default_caching = TTM_PL_FLAG_CACHED; 88 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; 89 #if IS_ENABLED(CONFIG_AGP) 90 if (rdev->flags & RADEON_IS_AGP) { 91 if (!rdev->ddev->agp) { 92 DRM_ERROR("AGP is not enabled for memory type %u\n", 93 (unsigned)type); 94 return -EINVAL; 95 } 96 if (!rdev->ddev->agp->cant_use_aperture) 97 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 98 man->available_caching = TTM_PL_FLAG_UNCACHED | 99 TTM_PL_FLAG_WC; 100 man->default_caching = TTM_PL_FLAG_WC; 101 } 102 #endif 103 break; 104 case TTM_PL_VRAM: 105 /* "On-card" video ram */ 106 man->func = &ttm_bo_manager_func; 107 man->gpu_offset = rdev->mc.vram_start; 108 man->flags = TTM_MEMTYPE_FLAG_FIXED | 109 TTM_MEMTYPE_FLAG_MAPPABLE; 110 man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; 111 man->default_caching = TTM_PL_FLAG_WC; 112 break; 113 default: 114 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); 115 return -EINVAL; 116 } 117 return 0; 118 } 119 120 static void radeon_evict_flags(struct ttm_buffer_object *bo, 121 struct ttm_placement *placement) 122 { 123 static const struct ttm_place placements = { 124 .fpfn = 0, 125 .lpfn = 0, 126 .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM 127 }; 128 129 struct radeon_bo *rbo; 130 131 if (!radeon_ttm_bo_is_radeon_bo(bo)) { 132 placement->placement = &placements; 133 placement->busy_placement = &placements; 134 placement->num_placement = 1; 135 placement->num_busy_placement = 1; 136 return; 137 } 138 rbo = container_of(bo, struct radeon_bo, tbo); 139 switch (bo->mem.mem_type) { 140 case TTM_PL_VRAM: 141 if (rbo->rdev->ring[radeon_copy_ring_index(rbo->rdev)].ready == false) 142 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 143 else if (rbo->rdev->mc.visible_vram_size < rbo->rdev->mc.real_vram_size && 144 bo->mem.start < (rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT)) { 145 unsigned fpfn = rbo->rdev->mc.visible_vram_size >> PAGE_SHIFT; 146 int i; 147 148 /* Try evicting to the CPU inaccessible part of VRAM 149 * first, but only set GTT as busy placement, so this 150 * BO will be evicted to GTT rather than causing other 151 * BOs to be evicted from VRAM 152 */ 153 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_VRAM | 154 RADEON_GEM_DOMAIN_GTT); 155 rbo->placement.num_busy_placement = 0; 156 for (i = 0; i < rbo->placement.num_placement; i++) { 157 if (rbo->placements[i].flags & TTM_PL_FLAG_VRAM) { 158 if (rbo->placements[i].fpfn < fpfn) 159 rbo->placements[i].fpfn = fpfn; 160 } else { 161 rbo->placement.busy_placement = 162 &rbo->placements[i]; 163 rbo->placement.num_busy_placement = 1; 164 } 165 } 166 } else 167 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT); 168 break; 169 case TTM_PL_TT: 170 default: 171 radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_CPU); 172 } 173 *placement = rbo->placement; 174 } 175 176 static int radeon_verify_access(struct ttm_buffer_object *bo, struct file *filp) 177 { 178 struct radeon_bo *rbo = container_of(bo, struct radeon_bo, tbo); 179 180 if (radeon_ttm_tt_has_userptr(bo->ttm)) 181 return -EPERM; 182 return drm_vma_node_verify_access(&rbo->tbo.base.vma_node, 183 filp->private_data); 184 } 185 186 static void radeon_move_null(struct ttm_buffer_object *bo, 187 struct ttm_mem_reg *new_mem) 188 { 189 struct ttm_mem_reg *old_mem = &bo->mem; 190 191 BUG_ON(old_mem->mm_node != NULL); 192 *old_mem = *new_mem; 193 new_mem->mm_node = NULL; 194 } 195 196 static int radeon_move_blit(struct ttm_buffer_object *bo, 197 bool evict, bool no_wait_gpu, 198 struct ttm_mem_reg *new_mem, 199 struct ttm_mem_reg *old_mem) 200 { 201 struct radeon_device *rdev; 202 uint64_t old_start, new_start; 203 struct radeon_fence *fence; 204 unsigned num_pages; 205 int r, ridx; 206 207 rdev = radeon_get_rdev(bo->bdev); 208 ridx = radeon_copy_ring_index(rdev); 209 old_start = (u64)old_mem->start << PAGE_SHIFT; 210 new_start = (u64)new_mem->start << PAGE_SHIFT; 211 212 switch (old_mem->mem_type) { 213 case TTM_PL_VRAM: 214 old_start += rdev->mc.vram_start; 215 break; 216 case TTM_PL_TT: 217 old_start += rdev->mc.gtt_start; 218 break; 219 default: 220 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 221 return -EINVAL; 222 } 223 switch (new_mem->mem_type) { 224 case TTM_PL_VRAM: 225 new_start += rdev->mc.vram_start; 226 break; 227 case TTM_PL_TT: 228 new_start += rdev->mc.gtt_start; 229 break; 230 default: 231 DRM_ERROR("Unknown placement %d\n", old_mem->mem_type); 232 return -EINVAL; 233 } 234 if (!rdev->ring[ridx].ready) { 235 DRM_ERROR("Trying to move memory with ring turned off.\n"); 236 return -EINVAL; 237 } 238 239 BUILD_BUG_ON((PAGE_SIZE % RADEON_GPU_PAGE_SIZE) != 0); 240 241 num_pages = new_mem->num_pages * (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); 242 fence = radeon_copy(rdev, old_start, new_start, num_pages, bo->base.resv); 243 if (IS_ERR(fence)) 244 return PTR_ERR(fence); 245 246 r = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, new_mem); 247 radeon_fence_unref(&fence); 248 return r; 249 } 250 251 static int radeon_move_vram_ram(struct ttm_buffer_object *bo, 252 bool evict, bool interruptible, 253 bool no_wait_gpu, 254 struct ttm_mem_reg *new_mem) 255 { 256 struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu }; 257 struct ttm_mem_reg *old_mem = &bo->mem; 258 struct ttm_mem_reg tmp_mem; 259 struct ttm_place placements; 260 struct ttm_placement placement; 261 int r; 262 263 tmp_mem = *new_mem; 264 tmp_mem.mm_node = NULL; 265 placement.num_placement = 1; 266 placement.placement = &placements; 267 placement.num_busy_placement = 1; 268 placement.busy_placement = &placements; 269 placements.fpfn = 0; 270 placements.lpfn = 0; 271 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 272 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, &ctx); 273 if (unlikely(r)) { 274 return r; 275 } 276 277 r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement); 278 if (unlikely(r)) { 279 goto out_cleanup; 280 } 281 282 r = ttm_tt_bind(bo->ttm, &tmp_mem, &ctx); 283 if (unlikely(r)) { 284 goto out_cleanup; 285 } 286 r = radeon_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem); 287 if (unlikely(r)) { 288 goto out_cleanup; 289 } 290 r = ttm_bo_move_ttm(bo, &ctx, new_mem); 291 out_cleanup: 292 ttm_bo_mem_put(bo, &tmp_mem); 293 return r; 294 } 295 296 static int radeon_move_ram_vram(struct ttm_buffer_object *bo, 297 bool evict, bool interruptible, 298 bool no_wait_gpu, 299 struct ttm_mem_reg *new_mem) 300 { 301 struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu }; 302 struct ttm_mem_reg *old_mem = &bo->mem; 303 struct ttm_mem_reg tmp_mem; 304 struct ttm_placement placement; 305 struct ttm_place placements; 306 int r; 307 308 tmp_mem = *new_mem; 309 tmp_mem.mm_node = NULL; 310 placement.num_placement = 1; 311 placement.placement = &placements; 312 placement.num_busy_placement = 1; 313 placement.busy_placement = &placements; 314 placements.fpfn = 0; 315 placements.lpfn = 0; 316 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 317 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, &ctx); 318 if (unlikely(r)) { 319 return r; 320 } 321 r = ttm_bo_move_ttm(bo, &ctx, &tmp_mem); 322 if (unlikely(r)) { 323 goto out_cleanup; 324 } 325 r = radeon_move_blit(bo, true, no_wait_gpu, new_mem, old_mem); 326 if (unlikely(r)) { 327 goto out_cleanup; 328 } 329 out_cleanup: 330 ttm_bo_mem_put(bo, &tmp_mem); 331 return r; 332 } 333 334 static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, 335 struct ttm_operation_ctx *ctx, 336 struct ttm_mem_reg *new_mem) 337 { 338 struct radeon_device *rdev; 339 struct radeon_bo *rbo; 340 struct ttm_mem_reg *old_mem = &bo->mem; 341 int r; 342 343 r = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); 344 if (r) 345 return r; 346 347 /* Can't move a pinned BO */ 348 rbo = container_of(bo, struct radeon_bo, tbo); 349 if (WARN_ON_ONCE(rbo->pin_count > 0)) 350 return -EINVAL; 351 352 rdev = radeon_get_rdev(bo->bdev); 353 if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 354 radeon_move_null(bo, new_mem); 355 return 0; 356 } 357 if ((old_mem->mem_type == TTM_PL_TT && 358 new_mem->mem_type == TTM_PL_SYSTEM) || 359 (old_mem->mem_type == TTM_PL_SYSTEM && 360 new_mem->mem_type == TTM_PL_TT)) { 361 /* bind is enough */ 362 radeon_move_null(bo, new_mem); 363 return 0; 364 } 365 if (!rdev->ring[radeon_copy_ring_index(rdev)].ready || 366 rdev->asic->copy.copy == NULL) { 367 /* use memcpy */ 368 goto memcpy; 369 } 370 371 if (old_mem->mem_type == TTM_PL_VRAM && 372 new_mem->mem_type == TTM_PL_SYSTEM) { 373 r = radeon_move_vram_ram(bo, evict, ctx->interruptible, 374 ctx->no_wait_gpu, new_mem); 375 } else if (old_mem->mem_type == TTM_PL_SYSTEM && 376 new_mem->mem_type == TTM_PL_VRAM) { 377 r = radeon_move_ram_vram(bo, evict, ctx->interruptible, 378 ctx->no_wait_gpu, new_mem); 379 } else { 380 r = radeon_move_blit(bo, evict, ctx->no_wait_gpu, 381 new_mem, old_mem); 382 } 383 384 if (r) { 385 memcpy: 386 r = ttm_bo_move_memcpy(bo, ctx, new_mem); 387 if (r) { 388 return r; 389 } 390 } 391 392 /* update statistics */ 393 atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &rdev->num_bytes_moved); 394 return 0; 395 } 396 397 static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 398 { 399 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 400 struct radeon_device *rdev = radeon_get_rdev(bdev); 401 402 mem->bus.addr = NULL; 403 mem->bus.offset = 0; 404 mem->bus.size = mem->num_pages << PAGE_SHIFT; 405 mem->bus.base = 0; 406 mem->bus.is_iomem = false; 407 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) 408 return -EINVAL; 409 switch (mem->mem_type) { 410 case TTM_PL_SYSTEM: 411 /* system memory */ 412 return 0; 413 case TTM_PL_TT: 414 #if IS_ENABLED(CONFIG_AGP) 415 if (rdev->flags & RADEON_IS_AGP) { 416 /* RADEON_IS_AGP is set only if AGP is active */ 417 mem->bus.offset = mem->start << PAGE_SHIFT; 418 mem->bus.base = rdev->mc.agp_base; 419 mem->bus.is_iomem = !rdev->ddev->agp->cant_use_aperture; 420 } 421 #endif 422 break; 423 case TTM_PL_VRAM: 424 mem->bus.offset = mem->start << PAGE_SHIFT; 425 /* check if it's visible */ 426 if ((mem->bus.offset + mem->bus.size) > rdev->mc.visible_vram_size) 427 return -EINVAL; 428 mem->bus.base = rdev->mc.aper_base; 429 mem->bus.is_iomem = true; 430 #ifdef __alpha__ 431 /* 432 * Alpha: use bus.addr to hold the ioremap() return, 433 * so we can modify bus.base below. 434 */ 435 if (mem->placement & TTM_PL_FLAG_WC) 436 mem->bus.addr = 437 ioremap_wc(mem->bus.base + mem->bus.offset, 438 mem->bus.size); 439 else 440 mem->bus.addr = 441 ioremap(mem->bus.base + mem->bus.offset, 442 mem->bus.size); 443 if (!mem->bus.addr) 444 return -ENOMEM; 445 446 /* 447 * Alpha: Use just the bus offset plus 448 * the hose/domain memory base for bus.base. 449 * It then can be used to build PTEs for VRAM 450 * access, as done in ttm_bo_vm_fault(). 451 */ 452 mem->bus.base = (mem->bus.base & 0x0ffffffffUL) + 453 rdev->ddev->hose->dense_mem_base; 454 #endif 455 break; 456 default: 457 return -EINVAL; 458 } 459 return 0; 460 } 461 462 static void radeon_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 463 { 464 } 465 466 /* 467 * TTM backend functions. 468 */ 469 struct radeon_ttm_tt { 470 struct ttm_dma_tt ttm; 471 struct radeon_device *rdev; 472 u64 offset; 473 474 uint64_t userptr; 475 struct mm_struct *usermm; 476 uint32_t userflags; 477 }; 478 479 /* prepare the sg table with the user pages */ 480 static int radeon_ttm_tt_pin_userptr(struct ttm_tt *ttm) 481 { 482 struct radeon_device *rdev = radeon_get_rdev(ttm->bdev); 483 struct radeon_ttm_tt *gtt = (void *)ttm; 484 unsigned pinned = 0, nents; 485 int r; 486 487 int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 488 enum dma_data_direction direction = write ? 489 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 490 491 if (current->mm != gtt->usermm) 492 return -EPERM; 493 494 if (gtt->userflags & RADEON_GEM_USERPTR_ANONONLY) { 495 /* check that we only pin down anonymous memory 496 to prevent problems with writeback */ 497 unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE; 498 struct vm_area_struct *vma; 499 vma = find_vma(gtt->usermm, gtt->userptr); 500 if (!vma || vma->vm_file || vma->vm_end < end) 501 return -EPERM; 502 } 503 504 do { 505 unsigned num_pages = ttm->num_pages - pinned; 506 uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE; 507 struct page **pages = ttm->pages + pinned; 508 509 r = get_user_pages(userptr, num_pages, write ? FOLL_WRITE : 0, 510 pages, NULL); 511 if (r < 0) 512 goto release_pages; 513 514 pinned += r; 515 516 } while (pinned < ttm->num_pages); 517 518 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, 519 ttm->num_pages << PAGE_SHIFT, 520 GFP_KERNEL); 521 if (r) 522 goto release_sg; 523 524 r = -ENOMEM; 525 nents = dma_map_sg(rdev->dev, ttm->sg->sgl, ttm->sg->nents, direction); 526 if (nents == 0) 527 goto release_sg; 528 529 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 530 gtt->ttm.dma_address, ttm->num_pages); 531 532 return 0; 533 534 release_sg: 535 kfree(ttm->sg); 536 537 release_pages: 538 release_pages(ttm->pages, pinned); 539 return r; 540 } 541 542 static void radeon_ttm_tt_unpin_userptr(struct ttm_tt *ttm) 543 { 544 struct radeon_device *rdev = radeon_get_rdev(ttm->bdev); 545 struct radeon_ttm_tt *gtt = (void *)ttm; 546 struct sg_page_iter sg_iter; 547 548 int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 549 enum dma_data_direction direction = write ? 550 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 551 552 /* double check that we don't free the table twice */ 553 if (!ttm->sg->sgl) 554 return; 555 556 /* free the sg table and pages again */ 557 dma_unmap_sg(rdev->dev, ttm->sg->sgl, ttm->sg->nents, direction); 558 559 for_each_sg_page(ttm->sg->sgl, &sg_iter, ttm->sg->nents, 0) { 560 struct page *page = sg_page_iter_page(&sg_iter); 561 if (!(gtt->userflags & RADEON_GEM_USERPTR_READONLY)) 562 set_page_dirty(page); 563 564 mark_page_accessed(page); 565 put_page(page); 566 } 567 568 sg_free_table(ttm->sg); 569 } 570 571 static int radeon_ttm_backend_bind(struct ttm_tt *ttm, 572 struct ttm_mem_reg *bo_mem) 573 { 574 struct radeon_ttm_tt *gtt = (void*)ttm; 575 uint32_t flags = RADEON_GART_PAGE_VALID | RADEON_GART_PAGE_READ | 576 RADEON_GART_PAGE_WRITE; 577 int r; 578 579 if (gtt->userptr) { 580 radeon_ttm_tt_pin_userptr(ttm); 581 flags &= ~RADEON_GART_PAGE_WRITE; 582 } 583 584 gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT); 585 if (!ttm->num_pages) { 586 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", 587 ttm->num_pages, bo_mem, ttm); 588 } 589 if (ttm->caching_state == tt_cached) 590 flags |= RADEON_GART_PAGE_SNOOP; 591 r = radeon_gart_bind(gtt->rdev, gtt->offset, ttm->num_pages, 592 ttm->pages, gtt->ttm.dma_address, flags); 593 if (r) { 594 DRM_ERROR("failed to bind %lu pages at 0x%08X\n", 595 ttm->num_pages, (unsigned)gtt->offset); 596 return r; 597 } 598 return 0; 599 } 600 601 static int radeon_ttm_backend_unbind(struct ttm_tt *ttm) 602 { 603 struct radeon_ttm_tt *gtt = (void *)ttm; 604 605 radeon_gart_unbind(gtt->rdev, gtt->offset, ttm->num_pages); 606 607 if (gtt->userptr) 608 radeon_ttm_tt_unpin_userptr(ttm); 609 610 return 0; 611 } 612 613 static void radeon_ttm_backend_destroy(struct ttm_tt *ttm) 614 { 615 struct radeon_ttm_tt *gtt = (void *)ttm; 616 617 ttm_dma_tt_fini(>t->ttm); 618 kfree(gtt); 619 } 620 621 static struct ttm_backend_func radeon_backend_func = { 622 .bind = &radeon_ttm_backend_bind, 623 .unbind = &radeon_ttm_backend_unbind, 624 .destroy = &radeon_ttm_backend_destroy, 625 }; 626 627 static struct ttm_tt *radeon_ttm_tt_create(struct ttm_buffer_object *bo, 628 uint32_t page_flags) 629 { 630 struct radeon_device *rdev; 631 struct radeon_ttm_tt *gtt; 632 633 rdev = radeon_get_rdev(bo->bdev); 634 #if IS_ENABLED(CONFIG_AGP) 635 if (rdev->flags & RADEON_IS_AGP) { 636 return ttm_agp_tt_create(bo, rdev->ddev->agp->bridge, 637 page_flags); 638 } 639 #endif 640 641 gtt = kzalloc(sizeof(struct radeon_ttm_tt), GFP_KERNEL); 642 if (gtt == NULL) { 643 return NULL; 644 } 645 gtt->ttm.ttm.func = &radeon_backend_func; 646 gtt->rdev = rdev; 647 if (ttm_dma_tt_init(>t->ttm, bo, page_flags)) { 648 kfree(gtt); 649 return NULL; 650 } 651 return >t->ttm.ttm; 652 } 653 654 static struct radeon_ttm_tt *radeon_ttm_tt_to_gtt(struct ttm_tt *ttm) 655 { 656 if (!ttm || ttm->func != &radeon_backend_func) 657 return NULL; 658 return (struct radeon_ttm_tt *)ttm; 659 } 660 661 static int radeon_ttm_tt_populate(struct ttm_tt *ttm, 662 struct ttm_operation_ctx *ctx) 663 { 664 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 665 struct radeon_device *rdev; 666 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 667 668 if (gtt && gtt->userptr) { 669 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); 670 if (!ttm->sg) 671 return -ENOMEM; 672 673 ttm->page_flags |= TTM_PAGE_FLAG_SG; 674 ttm->state = tt_unbound; 675 return 0; 676 } 677 678 if (slave && ttm->sg) { 679 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 680 gtt->ttm.dma_address, ttm->num_pages); 681 ttm->state = tt_unbound; 682 return 0; 683 } 684 685 rdev = radeon_get_rdev(ttm->bdev); 686 #if IS_ENABLED(CONFIG_AGP) 687 if (rdev->flags & RADEON_IS_AGP) { 688 return ttm_agp_tt_populate(ttm, ctx); 689 } 690 #endif 691 692 #ifdef CONFIG_SWIOTLB 693 if (rdev->need_swiotlb && swiotlb_nr_tbl()) { 694 return ttm_dma_populate(>t->ttm, rdev->dev, ctx); 695 } 696 #endif 697 698 return ttm_populate_and_map_pages(rdev->dev, >t->ttm, ctx); 699 } 700 701 static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm) 702 { 703 struct radeon_device *rdev; 704 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 705 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 706 707 if (gtt && gtt->userptr) { 708 kfree(ttm->sg); 709 ttm->page_flags &= ~TTM_PAGE_FLAG_SG; 710 return; 711 } 712 713 if (slave) 714 return; 715 716 rdev = radeon_get_rdev(ttm->bdev); 717 #if IS_ENABLED(CONFIG_AGP) 718 if (rdev->flags & RADEON_IS_AGP) { 719 ttm_agp_tt_unpopulate(ttm); 720 return; 721 } 722 #endif 723 724 #ifdef CONFIG_SWIOTLB 725 if (rdev->need_swiotlb && swiotlb_nr_tbl()) { 726 ttm_dma_unpopulate(>t->ttm, rdev->dev); 727 return; 728 } 729 #endif 730 731 ttm_unmap_and_unpopulate_pages(rdev->dev, >t->ttm); 732 } 733 734 int radeon_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr, 735 uint32_t flags) 736 { 737 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 738 739 if (gtt == NULL) 740 return -EINVAL; 741 742 gtt->userptr = addr; 743 gtt->usermm = current->mm; 744 gtt->userflags = flags; 745 return 0; 746 } 747 748 bool radeon_ttm_tt_has_userptr(struct ttm_tt *ttm) 749 { 750 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 751 752 if (gtt == NULL) 753 return false; 754 755 return !!gtt->userptr; 756 } 757 758 bool radeon_ttm_tt_is_readonly(struct ttm_tt *ttm) 759 { 760 struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm); 761 762 if (gtt == NULL) 763 return false; 764 765 return !!(gtt->userflags & RADEON_GEM_USERPTR_READONLY); 766 } 767 768 static struct ttm_bo_driver radeon_bo_driver = { 769 .ttm_tt_create = &radeon_ttm_tt_create, 770 .ttm_tt_populate = &radeon_ttm_tt_populate, 771 .ttm_tt_unpopulate = &radeon_ttm_tt_unpopulate, 772 .init_mem_type = &radeon_init_mem_type, 773 .eviction_valuable = ttm_bo_eviction_valuable, 774 .evict_flags = &radeon_evict_flags, 775 .move = &radeon_bo_move, 776 .verify_access = &radeon_verify_access, 777 .move_notify = &radeon_bo_move_notify, 778 .fault_reserve_notify = &radeon_bo_fault_reserve_notify, 779 .io_mem_reserve = &radeon_ttm_io_mem_reserve, 780 .io_mem_free = &radeon_ttm_io_mem_free, 781 }; 782 783 int radeon_ttm_init(struct radeon_device *rdev) 784 { 785 int r; 786 787 /* No others user of address space so set it to 0 */ 788 r = ttm_bo_device_init(&rdev->mman.bdev, 789 &radeon_bo_driver, 790 rdev->ddev->anon_inode->i_mapping, 791 rdev->ddev->vma_offset_manager, 792 dma_addressing_limited(&rdev->pdev->dev)); 793 if (r) { 794 DRM_ERROR("failed initializing buffer object driver(%d).\n", r); 795 return r; 796 } 797 rdev->mman.initialized = true; 798 r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_VRAM, 799 rdev->mc.real_vram_size >> PAGE_SHIFT); 800 if (r) { 801 DRM_ERROR("Failed initializing VRAM heap.\n"); 802 return r; 803 } 804 /* Change the size here instead of the init above so only lpfn is affected */ 805 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 806 807 r = radeon_bo_create(rdev, 256 * 1024, PAGE_SIZE, true, 808 RADEON_GEM_DOMAIN_VRAM, 0, NULL, 809 NULL, &rdev->stolen_vga_memory); 810 if (r) { 811 return r; 812 } 813 r = radeon_bo_reserve(rdev->stolen_vga_memory, false); 814 if (r) 815 return r; 816 r = radeon_bo_pin(rdev->stolen_vga_memory, RADEON_GEM_DOMAIN_VRAM, NULL); 817 radeon_bo_unreserve(rdev->stolen_vga_memory); 818 if (r) { 819 radeon_bo_unref(&rdev->stolen_vga_memory); 820 return r; 821 } 822 DRM_INFO("radeon: %uM of VRAM memory ready\n", 823 (unsigned) (rdev->mc.real_vram_size / (1024 * 1024))); 824 r = ttm_bo_init_mm(&rdev->mman.bdev, TTM_PL_TT, 825 rdev->mc.gtt_size >> PAGE_SHIFT); 826 if (r) { 827 DRM_ERROR("Failed initializing GTT heap.\n"); 828 return r; 829 } 830 DRM_INFO("radeon: %uM of GTT memory ready.\n", 831 (unsigned)(rdev->mc.gtt_size / (1024 * 1024))); 832 833 r = radeon_ttm_debugfs_init(rdev); 834 if (r) { 835 DRM_ERROR("Failed to init debugfs\n"); 836 return r; 837 } 838 return 0; 839 } 840 841 void radeon_ttm_fini(struct radeon_device *rdev) 842 { 843 int r; 844 845 if (!rdev->mman.initialized) 846 return; 847 radeon_ttm_debugfs_fini(rdev); 848 if (rdev->stolen_vga_memory) { 849 r = radeon_bo_reserve(rdev->stolen_vga_memory, false); 850 if (r == 0) { 851 radeon_bo_unpin(rdev->stolen_vga_memory); 852 radeon_bo_unreserve(rdev->stolen_vga_memory); 853 } 854 radeon_bo_unref(&rdev->stolen_vga_memory); 855 } 856 ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_VRAM); 857 ttm_bo_clean_mm(&rdev->mman.bdev, TTM_PL_TT); 858 ttm_bo_device_release(&rdev->mman.bdev); 859 radeon_gart_fini(rdev); 860 rdev->mman.initialized = false; 861 DRM_INFO("radeon: ttm finalized\n"); 862 } 863 864 /* this should only be called at bootup or when userspace 865 * isn't running */ 866 void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size) 867 { 868 struct ttm_mem_type_manager *man; 869 870 if (!rdev->mman.initialized) 871 return; 872 873 man = &rdev->mman.bdev.man[TTM_PL_VRAM]; 874 /* this just adjusts TTM size idea, which sets lpfn to the correct value */ 875 man->size = size >> PAGE_SHIFT; 876 } 877 878 static vm_fault_t radeon_ttm_fault(struct vm_fault *vmf) 879 { 880 struct ttm_buffer_object *bo; 881 struct radeon_device *rdev; 882 vm_fault_t ret; 883 884 bo = (struct ttm_buffer_object *)vmf->vma->vm_private_data; 885 if (bo == NULL) 886 return VM_FAULT_NOPAGE; 887 888 rdev = radeon_get_rdev(bo->bdev); 889 down_read(&rdev->pm.mclk_lock); 890 ret = ttm_bo_vm_fault(vmf); 891 up_read(&rdev->pm.mclk_lock); 892 return ret; 893 } 894 895 static struct vm_operations_struct radeon_ttm_vm_ops = { 896 .fault = radeon_ttm_fault, 897 .open = ttm_bo_vm_open, 898 .close = ttm_bo_vm_close, 899 .access = ttm_bo_vm_access 900 }; 901 902 int radeon_mmap(struct file *filp, struct vm_area_struct *vma) 903 { 904 int r; 905 struct drm_file *file_priv = filp->private_data; 906 struct radeon_device *rdev = file_priv->minor->dev->dev_private; 907 908 if (rdev == NULL) 909 return -EINVAL; 910 911 r = ttm_bo_mmap(filp, vma, &rdev->mman.bdev); 912 if (unlikely(r != 0)) 913 return r; 914 915 vma->vm_ops = &radeon_ttm_vm_ops; 916 return 0; 917 } 918 919 #if defined(CONFIG_DEBUG_FS) 920 921 static int radeon_mm_dump_table(struct seq_file *m, void *data) 922 { 923 struct drm_info_node *node = (struct drm_info_node *)m->private; 924 unsigned ttm_pl = *(int*)node->info_ent->data; 925 struct drm_device *dev = node->minor->dev; 926 struct radeon_device *rdev = dev->dev_private; 927 struct ttm_mem_type_manager *man = &rdev->mman.bdev.man[ttm_pl]; 928 struct drm_printer p = drm_seq_file_printer(m); 929 930 man->func->debug(man, &p); 931 return 0; 932 } 933 934 935 static int ttm_pl_vram = TTM_PL_VRAM; 936 static int ttm_pl_tt = TTM_PL_TT; 937 938 static struct drm_info_list radeon_ttm_debugfs_list[] = { 939 {"radeon_vram_mm", radeon_mm_dump_table, 0, &ttm_pl_vram}, 940 {"radeon_gtt_mm", radeon_mm_dump_table, 0, &ttm_pl_tt}, 941 {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL}, 942 #ifdef CONFIG_SWIOTLB 943 {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL} 944 #endif 945 }; 946 947 static int radeon_ttm_vram_open(struct inode *inode, struct file *filep) 948 { 949 struct radeon_device *rdev = inode->i_private; 950 i_size_write(inode, rdev->mc.mc_vram_size); 951 filep->private_data = inode->i_private; 952 return 0; 953 } 954 955 static ssize_t radeon_ttm_vram_read(struct file *f, char __user *buf, 956 size_t size, loff_t *pos) 957 { 958 struct radeon_device *rdev = f->private_data; 959 ssize_t result = 0; 960 int r; 961 962 if (size & 0x3 || *pos & 0x3) 963 return -EINVAL; 964 965 while (size) { 966 unsigned long flags; 967 uint32_t value; 968 969 if (*pos >= rdev->mc.mc_vram_size) 970 return result; 971 972 spin_lock_irqsave(&rdev->mmio_idx_lock, flags); 973 WREG32(RADEON_MM_INDEX, ((uint32_t)*pos) | 0x80000000); 974 if (rdev->family >= CHIP_CEDAR) 975 WREG32(EVERGREEN_MM_INDEX_HI, *pos >> 31); 976 value = RREG32(RADEON_MM_DATA); 977 spin_unlock_irqrestore(&rdev->mmio_idx_lock, flags); 978 979 r = put_user(value, (uint32_t *)buf); 980 if (r) 981 return r; 982 983 result += 4; 984 buf += 4; 985 *pos += 4; 986 size -= 4; 987 } 988 989 return result; 990 } 991 992 static const struct file_operations radeon_ttm_vram_fops = { 993 .owner = THIS_MODULE, 994 .open = radeon_ttm_vram_open, 995 .read = radeon_ttm_vram_read, 996 .llseek = default_llseek 997 }; 998 999 static int radeon_ttm_gtt_open(struct inode *inode, struct file *filep) 1000 { 1001 struct radeon_device *rdev = inode->i_private; 1002 i_size_write(inode, rdev->mc.gtt_size); 1003 filep->private_data = inode->i_private; 1004 return 0; 1005 } 1006 1007 static ssize_t radeon_ttm_gtt_read(struct file *f, char __user *buf, 1008 size_t size, loff_t *pos) 1009 { 1010 struct radeon_device *rdev = f->private_data; 1011 ssize_t result = 0; 1012 int r; 1013 1014 while (size) { 1015 loff_t p = *pos / PAGE_SIZE; 1016 unsigned off = *pos & ~PAGE_MASK; 1017 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); 1018 struct page *page; 1019 void *ptr; 1020 1021 if (p >= rdev->gart.num_cpu_pages) 1022 return result; 1023 1024 page = rdev->gart.pages[p]; 1025 if (page) { 1026 ptr = kmap(page); 1027 ptr += off; 1028 1029 r = copy_to_user(buf, ptr, cur_size); 1030 kunmap(rdev->gart.pages[p]); 1031 } else 1032 r = clear_user(buf, cur_size); 1033 1034 if (r) 1035 return -EFAULT; 1036 1037 result += cur_size; 1038 buf += cur_size; 1039 *pos += cur_size; 1040 size -= cur_size; 1041 } 1042 1043 return result; 1044 } 1045 1046 static const struct file_operations radeon_ttm_gtt_fops = { 1047 .owner = THIS_MODULE, 1048 .open = radeon_ttm_gtt_open, 1049 .read = radeon_ttm_gtt_read, 1050 .llseek = default_llseek 1051 }; 1052 1053 #endif 1054 1055 static int radeon_ttm_debugfs_init(struct radeon_device *rdev) 1056 { 1057 #if defined(CONFIG_DEBUG_FS) 1058 unsigned count; 1059 1060 struct drm_minor *minor = rdev->ddev->primary; 1061 struct dentry *root = minor->debugfs_root; 1062 1063 rdev->mman.vram = debugfs_create_file("radeon_vram", S_IFREG | S_IRUGO, 1064 root, rdev, 1065 &radeon_ttm_vram_fops); 1066 1067 rdev->mman.gtt = debugfs_create_file("radeon_gtt", S_IFREG | S_IRUGO, 1068 root, rdev, &radeon_ttm_gtt_fops); 1069 1070 count = ARRAY_SIZE(radeon_ttm_debugfs_list); 1071 1072 #ifdef CONFIG_SWIOTLB 1073 if (!(rdev->need_swiotlb && swiotlb_nr_tbl())) 1074 --count; 1075 #endif 1076 1077 return radeon_debugfs_add_files(rdev, radeon_ttm_debugfs_list, count); 1078 #else 1079 1080 return 0; 1081 #endif 1082 } 1083 1084 static void radeon_ttm_debugfs_fini(struct radeon_device *rdev) 1085 { 1086 #if defined(CONFIG_DEBUG_FS) 1087 1088 debugfs_remove(rdev->mman.vram); 1089 rdev->mman.vram = NULL; 1090 1091 debugfs_remove(rdev->mman.gtt); 1092 rdev->mman.gtt = NULL; 1093 #endif 1094 } 1095