1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 33 #include <linux/dma-mapping.h> 34 #include <linux/iommu.h> 35 #include <linux/hmm.h> 36 #include <linux/pagemap.h> 37 #include <linux/sched/task.h> 38 #include <linux/sched/mm.h> 39 #include <linux/seq_file.h> 40 #include <linux/slab.h> 41 #include <linux/swap.h> 42 #include <linux/swiotlb.h> 43 #include <linux/dma-buf.h> 44 #include <linux/sizes.h> 45 46 #include <drm/ttm/ttm_bo_api.h> 47 #include <drm/ttm/ttm_bo_driver.h> 48 #include <drm/ttm/ttm_placement.h> 49 #include <drm/ttm/ttm_module.h> 50 #include <drm/ttm/ttm_page_alloc.h> 51 52 #include <drm/drm_debugfs.h> 53 #include <drm/amdgpu_drm.h> 54 55 #include "amdgpu.h" 56 #include "amdgpu_object.h" 57 #include "amdgpu_trace.h" 58 #include "amdgpu_amdkfd.h" 59 #include "amdgpu_sdma.h" 60 #include "amdgpu_ras.h" 61 #include "amdgpu_atomfirmware.h" 62 #include "bif/bif_4_1_d.h" 63 64 #define AMDGPU_TTM_VRAM_MAX_DW_READ (size_t)128 65 66 static int amdgpu_ttm_backend_bind(struct ttm_bo_device *bdev, 67 struct ttm_tt *ttm, 68 struct ttm_resource *bo_mem); 69 70 static int amdgpu_ttm_init_on_chip(struct amdgpu_device *adev, 71 unsigned int type, 72 uint64_t size) 73 { 74 return ttm_range_man_init(&adev->mman.bdev, type, 75 false, size >> PAGE_SHIFT); 76 } 77 78 /** 79 * amdgpu_evict_flags - Compute placement flags 80 * 81 * @bo: The buffer object to evict 82 * @placement: Possible destination(s) for evicted BO 83 * 84 * Fill in placement data when ttm_bo_evict() is called 85 */ 86 static void amdgpu_evict_flags(struct ttm_buffer_object *bo, 87 struct ttm_placement *placement) 88 { 89 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 90 struct amdgpu_bo *abo; 91 static const struct ttm_place placements = { 92 .fpfn = 0, 93 .lpfn = 0, 94 .mem_type = TTM_PL_SYSTEM, 95 .flags = 0 96 }; 97 98 /* Don't handle scatter gather BOs */ 99 if (bo->type == ttm_bo_type_sg) { 100 placement->num_placement = 0; 101 placement->num_busy_placement = 0; 102 return; 103 } 104 105 /* Object isn't an AMDGPU object so ignore */ 106 if (!amdgpu_bo_is_amdgpu_bo(bo)) { 107 placement->placement = &placements; 108 placement->busy_placement = &placements; 109 placement->num_placement = 1; 110 placement->num_busy_placement = 1; 111 return; 112 } 113 114 abo = ttm_to_amdgpu_bo(bo); 115 switch (bo->mem.mem_type) { 116 case AMDGPU_PL_GDS: 117 case AMDGPU_PL_GWS: 118 case AMDGPU_PL_OA: 119 placement->num_placement = 0; 120 placement->num_busy_placement = 0; 121 return; 122 123 case TTM_PL_VRAM: 124 if (!adev->mman.buffer_funcs_enabled) { 125 /* Move to system memory */ 126 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); 127 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 128 !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) && 129 amdgpu_bo_in_cpu_visible_vram(abo)) { 130 131 /* Try evicting to the CPU inaccessible part of VRAM 132 * first, but only set GTT as busy placement, so this 133 * BO will be evicted to GTT rather than causing other 134 * BOs to be evicted from VRAM 135 */ 136 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | 137 AMDGPU_GEM_DOMAIN_GTT); 138 abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; 139 abo->placements[0].lpfn = 0; 140 abo->placement.busy_placement = &abo->placements[1]; 141 abo->placement.num_busy_placement = 1; 142 } else { 143 /* Move to GTT memory */ 144 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT); 145 } 146 break; 147 case TTM_PL_TT: 148 default: 149 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); 150 break; 151 } 152 *placement = abo->placement; 153 } 154 155 /** 156 * amdgpu_verify_access - Verify access for a mmap call 157 * 158 * @bo: The buffer object to map 159 * @filp: The file pointer from the process performing the mmap 160 * 161 * This is called by ttm_bo_mmap() to verify whether a process 162 * has the right to mmap a BO to their process space. 163 */ 164 static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp) 165 { 166 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 167 168 /* 169 * Don't verify access for KFD BOs. They don't have a GEM 170 * object associated with them. 171 */ 172 if (abo->kfd_bo) 173 return 0; 174 175 if (amdgpu_ttm_tt_get_usermm(bo->ttm)) 176 return -EPERM; 177 return drm_vma_node_verify_access(&abo->tbo.base.vma_node, 178 filp->private_data); 179 } 180 181 /** 182 * amdgpu_mm_node_addr - Compute the GPU relative offset of a GTT buffer. 183 * 184 * @bo: The bo to assign the memory to. 185 * @mm_node: Memory manager node for drm allocator. 186 * @mem: The region where the bo resides. 187 * 188 */ 189 static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo, 190 struct drm_mm_node *mm_node, 191 struct ttm_resource *mem) 192 { 193 uint64_t addr = 0; 194 195 if (mm_node->start != AMDGPU_BO_INVALID_OFFSET) { 196 addr = mm_node->start << PAGE_SHIFT; 197 addr += amdgpu_ttm_domain_start(amdgpu_ttm_adev(bo->bdev), 198 mem->mem_type); 199 } 200 return addr; 201 } 202 203 /** 204 * amdgpu_find_mm_node - Helper function finds the drm_mm_node corresponding to 205 * @offset. It also modifies the offset to be within the drm_mm_node returned 206 * 207 * @mem: The region where the bo resides. 208 * @offset: The offset that drm_mm_node is used for finding. 209 * 210 */ 211 static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_resource *mem, 212 uint64_t *offset) 213 { 214 struct drm_mm_node *mm_node = mem->mm_node; 215 216 while (*offset >= (mm_node->size << PAGE_SHIFT)) { 217 *offset -= (mm_node->size << PAGE_SHIFT); 218 ++mm_node; 219 } 220 return mm_node; 221 } 222 223 /** 224 * amdgpu_ttm_map_buffer - Map memory into the GART windows 225 * @bo: buffer object to map 226 * @mem: memory object to map 227 * @mm_node: drm_mm node object to map 228 * @num_pages: number of pages to map 229 * @offset: offset into @mm_node where to start 230 * @window: which GART window to use 231 * @ring: DMA ring to use for the copy 232 * @tmz: if we should setup a TMZ enabled mapping 233 * @addr: resulting address inside the MC address space 234 * 235 * Setup one of the GART windows to access a specific piece of memory or return 236 * the physical address for local memory. 237 */ 238 static int amdgpu_ttm_map_buffer(struct ttm_buffer_object *bo, 239 struct ttm_resource *mem, 240 struct drm_mm_node *mm_node, 241 unsigned num_pages, uint64_t offset, 242 unsigned window, struct amdgpu_ring *ring, 243 bool tmz, uint64_t *addr) 244 { 245 struct amdgpu_device *adev = ring->adev; 246 struct amdgpu_job *job; 247 unsigned num_dw, num_bytes; 248 struct dma_fence *fence; 249 uint64_t src_addr, dst_addr; 250 void *cpu_addr; 251 uint64_t flags; 252 unsigned int i; 253 int r; 254 255 BUG_ON(adev->mman.buffer_funcs->copy_max_bytes < 256 AMDGPU_GTT_MAX_TRANSFER_SIZE * 8); 257 258 /* Map only what can't be accessed directly */ 259 if (!tmz && mem->start != AMDGPU_BO_INVALID_OFFSET) { 260 *addr = amdgpu_mm_node_addr(bo, mm_node, mem) + offset; 261 return 0; 262 } 263 264 *addr = adev->gmc.gart_start; 265 *addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 266 AMDGPU_GPU_PAGE_SIZE; 267 *addr += offset & ~PAGE_MASK; 268 269 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); 270 num_bytes = num_pages * 8; 271 272 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes, 273 AMDGPU_IB_POOL_DELAYED, &job); 274 if (r) 275 return r; 276 277 src_addr = num_dw * 4; 278 src_addr += job->ibs[0].gpu_addr; 279 280 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo); 281 dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8; 282 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, 283 dst_addr, num_bytes, false); 284 285 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 286 WARN_ON(job->ibs[0].length_dw > num_dw); 287 288 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem); 289 if (tmz) 290 flags |= AMDGPU_PTE_TMZ; 291 292 cpu_addr = &job->ibs[0].ptr[num_dw]; 293 294 if (mem->mem_type == TTM_PL_TT) { 295 struct ttm_dma_tt *dma; 296 dma_addr_t *dma_address; 297 298 dma = container_of(bo->ttm, struct ttm_dma_tt, ttm); 299 dma_address = &dma->dma_address[offset >> PAGE_SHIFT]; 300 r = amdgpu_gart_map(adev, 0, num_pages, dma_address, flags, 301 cpu_addr); 302 if (r) 303 goto error_free; 304 } else { 305 dma_addr_t dma_address; 306 307 dma_address = (mm_node->start << PAGE_SHIFT) + offset; 308 dma_address += adev->vm_manager.vram_base_offset; 309 310 for (i = 0; i < num_pages; ++i) { 311 r = amdgpu_gart_map(adev, i << PAGE_SHIFT, 1, 312 &dma_address, flags, cpu_addr); 313 if (r) 314 goto error_free; 315 316 dma_address += PAGE_SIZE; 317 } 318 } 319 320 r = amdgpu_job_submit(job, &adev->mman.entity, 321 AMDGPU_FENCE_OWNER_UNDEFINED, &fence); 322 if (r) 323 goto error_free; 324 325 dma_fence_put(fence); 326 327 return r; 328 329 error_free: 330 amdgpu_job_free(job); 331 return r; 332 } 333 334 /** 335 * amdgpu_copy_ttm_mem_to_mem - Helper function for copy 336 * @adev: amdgpu device 337 * @src: buffer/address where to read from 338 * @dst: buffer/address where to write to 339 * @size: number of bytes to copy 340 * @tmz: if a secure copy should be used 341 * @resv: resv object to sync to 342 * @f: Returns the last fence if multiple jobs are submitted. 343 * 344 * The function copies @size bytes from {src->mem + src->offset} to 345 * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a 346 * move and different for a BO to BO copy. 347 * 348 */ 349 int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, 350 const struct amdgpu_copy_mem *src, 351 const struct amdgpu_copy_mem *dst, 352 uint64_t size, bool tmz, 353 struct dma_resv *resv, 354 struct dma_fence **f) 355 { 356 const uint32_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE * 357 AMDGPU_GPU_PAGE_SIZE); 358 359 uint64_t src_node_size, dst_node_size, src_offset, dst_offset; 360 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; 361 struct drm_mm_node *src_mm, *dst_mm; 362 struct dma_fence *fence = NULL; 363 int r = 0; 364 365 if (!adev->mman.buffer_funcs_enabled) { 366 DRM_ERROR("Trying to move memory with ring turned off.\n"); 367 return -EINVAL; 368 } 369 370 src_offset = src->offset; 371 if (src->mem->mm_node) { 372 src_mm = amdgpu_find_mm_node(src->mem, &src_offset); 373 src_node_size = (src_mm->size << PAGE_SHIFT) - src_offset; 374 } else { 375 src_mm = NULL; 376 src_node_size = ULLONG_MAX; 377 } 378 379 dst_offset = dst->offset; 380 if (dst->mem->mm_node) { 381 dst_mm = amdgpu_find_mm_node(dst->mem, &dst_offset); 382 dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst_offset; 383 } else { 384 dst_mm = NULL; 385 dst_node_size = ULLONG_MAX; 386 } 387 388 mutex_lock(&adev->mman.gtt_window_lock); 389 390 while (size) { 391 uint32_t src_page_offset = src_offset & ~PAGE_MASK; 392 uint32_t dst_page_offset = dst_offset & ~PAGE_MASK; 393 struct dma_fence *next; 394 uint32_t cur_size; 395 uint64_t from, to; 396 397 /* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst 398 * begins at an offset, then adjust the size accordingly 399 */ 400 cur_size = max(src_page_offset, dst_page_offset); 401 cur_size = min(min3(src_node_size, dst_node_size, size), 402 (uint64_t)(GTT_MAX_BYTES - cur_size)); 403 404 /* Map src to window 0 and dst to window 1. */ 405 r = amdgpu_ttm_map_buffer(src->bo, src->mem, src_mm, 406 PFN_UP(cur_size + src_page_offset), 407 src_offset, 0, ring, tmz, &from); 408 if (r) 409 goto error; 410 411 r = amdgpu_ttm_map_buffer(dst->bo, dst->mem, dst_mm, 412 PFN_UP(cur_size + dst_page_offset), 413 dst_offset, 1, ring, tmz, &to); 414 if (r) 415 goto error; 416 417 r = amdgpu_copy_buffer(ring, from, to, cur_size, 418 resv, &next, false, true, tmz); 419 if (r) 420 goto error; 421 422 dma_fence_put(fence); 423 fence = next; 424 425 size -= cur_size; 426 if (!size) 427 break; 428 429 src_node_size -= cur_size; 430 if (!src_node_size) { 431 ++src_mm; 432 src_node_size = src_mm->size << PAGE_SHIFT; 433 src_offset = 0; 434 } else { 435 src_offset += cur_size; 436 } 437 438 dst_node_size -= cur_size; 439 if (!dst_node_size) { 440 ++dst_mm; 441 dst_node_size = dst_mm->size << PAGE_SHIFT; 442 dst_offset = 0; 443 } else { 444 dst_offset += cur_size; 445 } 446 } 447 error: 448 mutex_unlock(&adev->mman.gtt_window_lock); 449 if (f) 450 *f = dma_fence_get(fence); 451 dma_fence_put(fence); 452 return r; 453 } 454 455 /** 456 * amdgpu_move_blit - Copy an entire buffer to another buffer 457 * 458 * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to 459 * help move buffers to and from VRAM. 460 */ 461 static int amdgpu_move_blit(struct ttm_buffer_object *bo, 462 bool evict, 463 struct ttm_resource *new_mem, 464 struct ttm_resource *old_mem) 465 { 466 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 467 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 468 struct amdgpu_copy_mem src, dst; 469 struct dma_fence *fence = NULL; 470 int r; 471 472 src.bo = bo; 473 dst.bo = bo; 474 src.mem = old_mem; 475 dst.mem = new_mem; 476 src.offset = 0; 477 dst.offset = 0; 478 479 r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst, 480 new_mem->num_pages << PAGE_SHIFT, 481 amdgpu_bo_encrypted(abo), 482 bo->base.resv, &fence); 483 if (r) 484 goto error; 485 486 /* clear the space being freed */ 487 if (old_mem->mem_type == TTM_PL_VRAM && 488 (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) { 489 struct dma_fence *wipe_fence = NULL; 490 491 r = amdgpu_fill_buffer(ttm_to_amdgpu_bo(bo), AMDGPU_POISON, 492 NULL, &wipe_fence); 493 if (r) { 494 goto error; 495 } else if (wipe_fence) { 496 dma_fence_put(fence); 497 fence = wipe_fence; 498 } 499 } 500 501 /* Always block for VM page tables before committing the new location */ 502 if (bo->type == ttm_bo_type_kernel) 503 r = ttm_bo_move_accel_cleanup(bo, fence, true, false, new_mem); 504 else 505 r = ttm_bo_move_accel_cleanup(bo, fence, evict, true, new_mem); 506 dma_fence_put(fence); 507 return r; 508 509 error: 510 if (fence) 511 dma_fence_wait(fence, false); 512 dma_fence_put(fence); 513 return r; 514 } 515 516 /** 517 * amdgpu_move_vram_ram - Copy VRAM buffer to RAM buffer 518 * 519 * Called by amdgpu_bo_move(). 520 */ 521 static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict, 522 struct ttm_operation_ctx *ctx, 523 struct ttm_resource *new_mem) 524 { 525 struct ttm_resource *old_mem = &bo->mem; 526 struct ttm_resource tmp_mem; 527 struct ttm_place placements; 528 struct ttm_placement placement; 529 int r; 530 531 /* create space/pages for new_mem in GTT space */ 532 tmp_mem = *new_mem; 533 tmp_mem.mm_node = NULL; 534 placement.num_placement = 1; 535 placement.placement = &placements; 536 placement.num_busy_placement = 1; 537 placement.busy_placement = &placements; 538 placements.fpfn = 0; 539 placements.lpfn = 0; 540 placements.mem_type = TTM_PL_TT; 541 placements.flags = 0; 542 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx); 543 if (unlikely(r)) { 544 pr_err("Failed to find GTT space for blit from VRAM\n"); 545 return r; 546 } 547 548 r = ttm_tt_populate(bo->bdev, bo->ttm, ctx); 549 if (unlikely(r)) 550 goto out_cleanup; 551 552 /* Bind the memory to the GTT space */ 553 r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, &tmp_mem); 554 if (unlikely(r)) { 555 goto out_cleanup; 556 } 557 558 /* blit VRAM to GTT */ 559 r = amdgpu_move_blit(bo, evict, &tmp_mem, old_mem); 560 if (unlikely(r)) { 561 goto out_cleanup; 562 } 563 564 /* move BO (in tmp_mem) to new_mem */ 565 r = ttm_bo_move_ttm(bo, ctx, new_mem); 566 out_cleanup: 567 ttm_resource_free(bo, &tmp_mem); 568 return r; 569 } 570 571 /** 572 * amdgpu_move_ram_vram - Copy buffer from RAM to VRAM 573 * 574 * Called by amdgpu_bo_move(). 575 */ 576 static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict, 577 struct ttm_operation_ctx *ctx, 578 struct ttm_resource *new_mem) 579 { 580 struct ttm_resource *old_mem = &bo->mem; 581 struct ttm_resource tmp_mem; 582 struct ttm_placement placement; 583 struct ttm_place placements; 584 int r; 585 586 /* make space in GTT for old_mem buffer */ 587 tmp_mem = *new_mem; 588 tmp_mem.mm_node = NULL; 589 placement.num_placement = 1; 590 placement.placement = &placements; 591 placement.num_busy_placement = 1; 592 placement.busy_placement = &placements; 593 placements.fpfn = 0; 594 placements.lpfn = 0; 595 placements.mem_type = TTM_PL_TT; 596 placements.flags = 0; 597 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx); 598 if (unlikely(r)) { 599 pr_err("Failed to find GTT space for blit to VRAM\n"); 600 return r; 601 } 602 603 /* move/bind old memory to GTT space */ 604 r = ttm_bo_move_ttm(bo, ctx, &tmp_mem); 605 if (unlikely(r)) { 606 goto out_cleanup; 607 } 608 609 /* copy to VRAM */ 610 r = amdgpu_move_blit(bo, evict, new_mem, old_mem); 611 if (unlikely(r)) { 612 goto out_cleanup; 613 } 614 out_cleanup: 615 ttm_resource_free(bo, &tmp_mem); 616 return r; 617 } 618 619 /** 620 * amdgpu_mem_visible - Check that memory can be accessed by ttm_bo_move_memcpy 621 * 622 * Called by amdgpu_bo_move() 623 */ 624 static bool amdgpu_mem_visible(struct amdgpu_device *adev, 625 struct ttm_resource *mem) 626 { 627 struct drm_mm_node *nodes = mem->mm_node; 628 629 if (mem->mem_type == TTM_PL_SYSTEM || 630 mem->mem_type == TTM_PL_TT) 631 return true; 632 if (mem->mem_type != TTM_PL_VRAM) 633 return false; 634 635 /* ttm_resource_ioremap only supports contiguous memory */ 636 if (nodes->size != mem->num_pages) 637 return false; 638 639 return ((nodes->start + nodes->size) << PAGE_SHIFT) 640 <= adev->gmc.visible_vram_size; 641 } 642 643 /** 644 * amdgpu_bo_move - Move a buffer object to a new memory location 645 * 646 * Called by ttm_bo_handle_move_mem() 647 */ 648 static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, 649 struct ttm_operation_ctx *ctx, 650 struct ttm_resource *new_mem) 651 { 652 struct amdgpu_device *adev; 653 struct amdgpu_bo *abo; 654 struct ttm_resource *old_mem = &bo->mem; 655 int r; 656 657 /* Can't move a pinned BO */ 658 abo = ttm_to_amdgpu_bo(bo); 659 if (WARN_ON_ONCE(abo->tbo.pin_count > 0)) 660 return -EINVAL; 661 662 adev = amdgpu_ttm_adev(bo->bdev); 663 664 if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 665 ttm_bo_move_null(bo, new_mem); 666 return 0; 667 } 668 if (old_mem->mem_type == TTM_PL_SYSTEM && 669 new_mem->mem_type == TTM_PL_TT) { 670 ttm_bo_move_null(bo, new_mem); 671 return 0; 672 } 673 674 if (old_mem->mem_type == TTM_PL_TT && 675 new_mem->mem_type == TTM_PL_SYSTEM) 676 return ttm_bo_move_ttm(bo, ctx, new_mem); 677 678 if (old_mem->mem_type == AMDGPU_PL_GDS || 679 old_mem->mem_type == AMDGPU_PL_GWS || 680 old_mem->mem_type == AMDGPU_PL_OA || 681 new_mem->mem_type == AMDGPU_PL_GDS || 682 new_mem->mem_type == AMDGPU_PL_GWS || 683 new_mem->mem_type == AMDGPU_PL_OA) { 684 /* Nothing to save here */ 685 ttm_bo_move_null(bo, new_mem); 686 return 0; 687 } 688 689 if (!adev->mman.buffer_funcs_enabled) { 690 r = -ENODEV; 691 goto memcpy; 692 } 693 694 if (old_mem->mem_type == TTM_PL_VRAM && 695 new_mem->mem_type == TTM_PL_SYSTEM) { 696 r = amdgpu_move_vram_ram(bo, evict, ctx, new_mem); 697 } else if (old_mem->mem_type == TTM_PL_SYSTEM && 698 new_mem->mem_type == TTM_PL_VRAM) { 699 r = amdgpu_move_ram_vram(bo, evict, ctx, new_mem); 700 } else { 701 r = amdgpu_move_blit(bo, evict, 702 new_mem, old_mem); 703 } 704 705 if (r) { 706 memcpy: 707 /* Check that all memory is CPU accessible */ 708 if (!amdgpu_mem_visible(adev, old_mem) || 709 !amdgpu_mem_visible(adev, new_mem)) { 710 pr_err("Move buffer fallback to memcpy unavailable\n"); 711 return r; 712 } 713 714 r = ttm_bo_move_memcpy(bo, ctx, new_mem); 715 if (r) 716 return r; 717 } 718 719 if (bo->type == ttm_bo_type_device && 720 new_mem->mem_type == TTM_PL_VRAM && 721 old_mem->mem_type != TTM_PL_VRAM) { 722 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU 723 * accesses the BO after it's moved. 724 */ 725 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 726 } 727 728 /* update statistics */ 729 atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved); 730 return 0; 731 } 732 733 /** 734 * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault 735 * 736 * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault() 737 */ 738 static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_resource *mem) 739 { 740 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 741 struct drm_mm_node *mm_node = mem->mm_node; 742 size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT; 743 744 switch (mem->mem_type) { 745 case TTM_PL_SYSTEM: 746 /* system memory */ 747 return 0; 748 case TTM_PL_TT: 749 break; 750 case TTM_PL_VRAM: 751 mem->bus.offset = mem->start << PAGE_SHIFT; 752 /* check if it's visible */ 753 if ((mem->bus.offset + bus_size) > adev->gmc.visible_vram_size) 754 return -EINVAL; 755 /* Only physically contiguous buffers apply. In a contiguous 756 * buffer, size of the first mm_node would match the number of 757 * pages in ttm_resource. 758 */ 759 if (adev->mman.aper_base_kaddr && 760 (mm_node->size == mem->num_pages)) 761 mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr + 762 mem->bus.offset; 763 764 mem->bus.offset += adev->gmc.aper_base; 765 mem->bus.is_iomem = true; 766 mem->bus.caching = ttm_write_combined; 767 break; 768 default: 769 return -EINVAL; 770 } 771 return 0; 772 } 773 774 static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo, 775 unsigned long page_offset) 776 { 777 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 778 uint64_t offset = (page_offset << PAGE_SHIFT); 779 struct drm_mm_node *mm; 780 781 mm = amdgpu_find_mm_node(&bo->mem, &offset); 782 offset += adev->gmc.aper_base; 783 return mm->start + (offset >> PAGE_SHIFT); 784 } 785 786 /** 787 * amdgpu_ttm_domain_start - Returns GPU start address 788 * @adev: amdgpu device object 789 * @type: type of the memory 790 * 791 * Returns: 792 * GPU start address of a memory domain 793 */ 794 795 uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type) 796 { 797 switch (type) { 798 case TTM_PL_TT: 799 return adev->gmc.gart_start; 800 case TTM_PL_VRAM: 801 return adev->gmc.vram_start; 802 } 803 804 return 0; 805 } 806 807 /* 808 * TTM backend functions. 809 */ 810 struct amdgpu_ttm_tt { 811 struct ttm_dma_tt ttm; 812 struct drm_gem_object *gobj; 813 u64 offset; 814 uint64_t userptr; 815 struct task_struct *usertask; 816 uint32_t userflags; 817 bool bound; 818 #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR) 819 struct hmm_range *range; 820 #endif 821 }; 822 823 #ifdef CONFIG_DRM_AMDGPU_USERPTR 824 /** 825 * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user 826 * memory and start HMM tracking CPU page table update 827 * 828 * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only 829 * once afterwards to stop HMM tracking 830 */ 831 int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages) 832 { 833 struct ttm_tt *ttm = bo->tbo.ttm; 834 struct amdgpu_ttm_tt *gtt = (void *)ttm; 835 unsigned long start = gtt->userptr; 836 struct vm_area_struct *vma; 837 struct hmm_range *range; 838 unsigned long timeout; 839 struct mm_struct *mm; 840 unsigned long i; 841 int r = 0; 842 843 mm = bo->notifier.mm; 844 if (unlikely(!mm)) { 845 DRM_DEBUG_DRIVER("BO is not registered?\n"); 846 return -EFAULT; 847 } 848 849 /* Another get_user_pages is running at the same time?? */ 850 if (WARN_ON(gtt->range)) 851 return -EFAULT; 852 853 if (!mmget_not_zero(mm)) /* Happens during process shutdown */ 854 return -ESRCH; 855 856 range = kzalloc(sizeof(*range), GFP_KERNEL); 857 if (unlikely(!range)) { 858 r = -ENOMEM; 859 goto out; 860 } 861 range->notifier = &bo->notifier; 862 range->start = bo->notifier.interval_tree.start; 863 range->end = bo->notifier.interval_tree.last + 1; 864 range->default_flags = HMM_PFN_REQ_FAULT; 865 if (!amdgpu_ttm_tt_is_readonly(ttm)) 866 range->default_flags |= HMM_PFN_REQ_WRITE; 867 868 range->hmm_pfns = kvmalloc_array(ttm->num_pages, 869 sizeof(*range->hmm_pfns), GFP_KERNEL); 870 if (unlikely(!range->hmm_pfns)) { 871 r = -ENOMEM; 872 goto out_free_ranges; 873 } 874 875 mmap_read_lock(mm); 876 vma = find_vma(mm, start); 877 if (unlikely(!vma || start < vma->vm_start)) { 878 r = -EFAULT; 879 goto out_unlock; 880 } 881 if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) && 882 vma->vm_file)) { 883 r = -EPERM; 884 goto out_unlock; 885 } 886 mmap_read_unlock(mm); 887 timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 888 889 retry: 890 range->notifier_seq = mmu_interval_read_begin(&bo->notifier); 891 892 mmap_read_lock(mm); 893 r = hmm_range_fault(range); 894 mmap_read_unlock(mm); 895 if (unlikely(r)) { 896 /* 897 * FIXME: This timeout should encompass the retry from 898 * mmu_interval_read_retry() as well. 899 */ 900 if (r == -EBUSY && !time_after(jiffies, timeout)) 901 goto retry; 902 goto out_free_pfns; 903 } 904 905 /* 906 * Due to default_flags, all pages are HMM_PFN_VALID or 907 * hmm_range_fault() fails. FIXME: The pages cannot be touched outside 908 * the notifier_lock, and mmu_interval_read_retry() must be done first. 909 */ 910 for (i = 0; i < ttm->num_pages; i++) 911 pages[i] = hmm_pfn_to_page(range->hmm_pfns[i]); 912 913 gtt->range = range; 914 mmput(mm); 915 916 return 0; 917 918 out_unlock: 919 mmap_read_unlock(mm); 920 out_free_pfns: 921 kvfree(range->hmm_pfns); 922 out_free_ranges: 923 kfree(range); 924 out: 925 mmput(mm); 926 return r; 927 } 928 929 /** 930 * amdgpu_ttm_tt_userptr_range_done - stop HMM track the CPU page table change 931 * Check if the pages backing this ttm range have been invalidated 932 * 933 * Returns: true if pages are still valid 934 */ 935 bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm) 936 { 937 struct amdgpu_ttm_tt *gtt = (void *)ttm; 938 bool r = false; 939 940 if (!gtt || !gtt->userptr) 941 return false; 942 943 DRM_DEBUG_DRIVER("user_pages_done 0x%llx pages 0x%lx\n", 944 gtt->userptr, ttm->num_pages); 945 946 WARN_ONCE(!gtt->range || !gtt->range->hmm_pfns, 947 "No user pages to check\n"); 948 949 if (gtt->range) { 950 /* 951 * FIXME: Must always hold notifier_lock for this, and must 952 * not ignore the return code. 953 */ 954 r = mmu_interval_read_retry(gtt->range->notifier, 955 gtt->range->notifier_seq); 956 kvfree(gtt->range->hmm_pfns); 957 kfree(gtt->range); 958 gtt->range = NULL; 959 } 960 961 return !r; 962 } 963 #endif 964 965 /** 966 * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary. 967 * 968 * Called by amdgpu_cs_list_validate(). This creates the page list 969 * that backs user memory and will ultimately be mapped into the device 970 * address space. 971 */ 972 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages) 973 { 974 unsigned long i; 975 976 for (i = 0; i < ttm->num_pages; ++i) 977 ttm->pages[i] = pages ? pages[i] : NULL; 978 } 979 980 /** 981 * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages 982 * 983 * Called by amdgpu_ttm_backend_bind() 984 **/ 985 static int amdgpu_ttm_tt_pin_userptr(struct ttm_bo_device *bdev, 986 struct ttm_tt *ttm) 987 { 988 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 989 struct amdgpu_ttm_tt *gtt = (void *)ttm; 990 int r; 991 992 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); 993 enum dma_data_direction direction = write ? 994 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 995 996 /* Allocate an SG array and squash pages into it */ 997 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, 998 ttm->num_pages << PAGE_SHIFT, 999 GFP_KERNEL); 1000 if (r) 1001 goto release_sg; 1002 1003 /* Map SG to device */ 1004 r = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); 1005 if (r) 1006 goto release_sg; 1007 1008 /* convert SG to linear array of pages and dma addresses */ 1009 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 1010 gtt->ttm.dma_address, ttm->num_pages); 1011 1012 return 0; 1013 1014 release_sg: 1015 kfree(ttm->sg); 1016 return r; 1017 } 1018 1019 /** 1020 * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages 1021 */ 1022 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_bo_device *bdev, 1023 struct ttm_tt *ttm) 1024 { 1025 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 1026 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1027 1028 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); 1029 enum dma_data_direction direction = write ? 1030 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 1031 1032 /* double check that we don't free the table twice */ 1033 if (!ttm->sg->sgl) 1034 return; 1035 1036 /* unmap the pages mapped to the device */ 1037 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); 1038 sg_free_table(ttm->sg); 1039 1040 #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR) 1041 if (gtt->range) { 1042 unsigned long i; 1043 1044 for (i = 0; i < ttm->num_pages; i++) { 1045 if (ttm->pages[i] != 1046 hmm_pfn_to_page(gtt->range->hmm_pfns[i])) 1047 break; 1048 } 1049 1050 WARN((i == ttm->num_pages), "Missing get_user_page_done\n"); 1051 } 1052 #endif 1053 } 1054 1055 static int amdgpu_ttm_gart_bind(struct amdgpu_device *adev, 1056 struct ttm_buffer_object *tbo, 1057 uint64_t flags) 1058 { 1059 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo); 1060 struct ttm_tt *ttm = tbo->ttm; 1061 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1062 int r; 1063 1064 if (amdgpu_bo_encrypted(abo)) 1065 flags |= AMDGPU_PTE_TMZ; 1066 1067 if (abo->flags & AMDGPU_GEM_CREATE_CP_MQD_GFX9) { 1068 uint64_t page_idx = 1; 1069 1070 r = amdgpu_gart_bind(adev, gtt->offset, page_idx, 1071 ttm->pages, gtt->ttm.dma_address, flags); 1072 if (r) 1073 goto gart_bind_fail; 1074 1075 /* The memory type of the first page defaults to UC. Now 1076 * modify the memory type to NC from the second page of 1077 * the BO onward. 1078 */ 1079 flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; 1080 flags |= AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_NC); 1081 1082 r = amdgpu_gart_bind(adev, 1083 gtt->offset + (page_idx << PAGE_SHIFT), 1084 ttm->num_pages - page_idx, 1085 &ttm->pages[page_idx], 1086 &(gtt->ttm.dma_address[page_idx]), flags); 1087 } else { 1088 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, 1089 ttm->pages, gtt->ttm.dma_address, flags); 1090 } 1091 1092 gart_bind_fail: 1093 if (r) 1094 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n", 1095 ttm->num_pages, gtt->offset); 1096 1097 return r; 1098 } 1099 1100 /** 1101 * amdgpu_ttm_backend_bind - Bind GTT memory 1102 * 1103 * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem(). 1104 * This handles binding GTT memory to the device address space. 1105 */ 1106 static int amdgpu_ttm_backend_bind(struct ttm_bo_device *bdev, 1107 struct ttm_tt *ttm, 1108 struct ttm_resource *bo_mem) 1109 { 1110 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 1111 struct amdgpu_ttm_tt *gtt = (void*)ttm; 1112 uint64_t flags; 1113 int r = 0; 1114 1115 if (!bo_mem) 1116 return -EINVAL; 1117 1118 if (gtt->bound) 1119 return 0; 1120 1121 if (gtt->userptr) { 1122 r = amdgpu_ttm_tt_pin_userptr(bdev, ttm); 1123 if (r) { 1124 DRM_ERROR("failed to pin userptr\n"); 1125 return r; 1126 } 1127 } 1128 if (!ttm->num_pages) { 1129 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", 1130 ttm->num_pages, bo_mem, ttm); 1131 } 1132 1133 if (bo_mem->mem_type == AMDGPU_PL_GDS || 1134 bo_mem->mem_type == AMDGPU_PL_GWS || 1135 bo_mem->mem_type == AMDGPU_PL_OA) 1136 return -EINVAL; 1137 1138 if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) { 1139 gtt->offset = AMDGPU_BO_INVALID_OFFSET; 1140 return 0; 1141 } 1142 1143 /* compute PTE flags relevant to this BO memory */ 1144 flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem); 1145 1146 /* bind pages into GART page tables */ 1147 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT; 1148 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, 1149 ttm->pages, gtt->ttm.dma_address, flags); 1150 1151 if (r) 1152 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n", 1153 ttm->num_pages, gtt->offset); 1154 gtt->bound = true; 1155 return r; 1156 } 1157 1158 /** 1159 * amdgpu_ttm_alloc_gart - Allocate GART memory for buffer object 1160 */ 1161 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo) 1162 { 1163 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1164 struct ttm_operation_ctx ctx = { false, false }; 1165 struct amdgpu_ttm_tt *gtt = (void*)bo->ttm; 1166 struct ttm_resource tmp; 1167 struct ttm_placement placement; 1168 struct ttm_place placements; 1169 uint64_t addr, flags; 1170 int r; 1171 1172 if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET) 1173 return 0; 1174 1175 addr = amdgpu_gmc_agp_addr(bo); 1176 if (addr != AMDGPU_BO_INVALID_OFFSET) { 1177 bo->mem.start = addr >> PAGE_SHIFT; 1178 } else { 1179 1180 /* allocate GART space */ 1181 tmp = bo->mem; 1182 tmp.mm_node = NULL; 1183 placement.num_placement = 1; 1184 placement.placement = &placements; 1185 placement.num_busy_placement = 1; 1186 placement.busy_placement = &placements; 1187 placements.fpfn = 0; 1188 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT; 1189 placements.mem_type = TTM_PL_TT; 1190 placements.flags = bo->mem.placement; 1191 1192 r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx); 1193 if (unlikely(r)) 1194 return r; 1195 1196 /* compute PTE flags for this buffer object */ 1197 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp); 1198 1199 /* Bind pages */ 1200 gtt->offset = (u64)tmp.start << PAGE_SHIFT; 1201 r = amdgpu_ttm_gart_bind(adev, bo, flags); 1202 if (unlikely(r)) { 1203 ttm_resource_free(bo, &tmp); 1204 return r; 1205 } 1206 1207 ttm_resource_free(bo, &bo->mem); 1208 bo->mem = tmp; 1209 } 1210 1211 return 0; 1212 } 1213 1214 /** 1215 * amdgpu_ttm_recover_gart - Rebind GTT pages 1216 * 1217 * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to 1218 * rebind GTT pages during a GPU reset. 1219 */ 1220 int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo) 1221 { 1222 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); 1223 uint64_t flags; 1224 int r; 1225 1226 if (!tbo->ttm) 1227 return 0; 1228 1229 flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem); 1230 r = amdgpu_ttm_gart_bind(adev, tbo, flags); 1231 1232 return r; 1233 } 1234 1235 /** 1236 * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages 1237 * 1238 * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and 1239 * ttm_tt_destroy(). 1240 */ 1241 static void amdgpu_ttm_backend_unbind(struct ttm_bo_device *bdev, 1242 struct ttm_tt *ttm) 1243 { 1244 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 1245 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1246 int r; 1247 1248 if (!gtt->bound) 1249 return; 1250 1251 /* if the pages have userptr pinning then clear that first */ 1252 if (gtt->userptr) 1253 amdgpu_ttm_tt_unpin_userptr(bdev, ttm); 1254 1255 if (gtt->offset == AMDGPU_BO_INVALID_OFFSET) 1256 return; 1257 1258 /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */ 1259 r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages); 1260 if (r) 1261 DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n", 1262 gtt->ttm.ttm.num_pages, gtt->offset); 1263 gtt->bound = false; 1264 } 1265 1266 static void amdgpu_ttm_backend_destroy(struct ttm_bo_device *bdev, 1267 struct ttm_tt *ttm) 1268 { 1269 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1270 1271 amdgpu_ttm_backend_unbind(bdev, ttm); 1272 ttm_tt_destroy_common(bdev, ttm); 1273 if (gtt->usertask) 1274 put_task_struct(gtt->usertask); 1275 1276 ttm_dma_tt_fini(>t->ttm); 1277 kfree(gtt); 1278 } 1279 1280 /** 1281 * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO 1282 * 1283 * @bo: The buffer object to create a GTT ttm_tt object around 1284 * 1285 * Called by ttm_tt_create(). 1286 */ 1287 static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo, 1288 uint32_t page_flags) 1289 { 1290 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 1291 struct amdgpu_ttm_tt *gtt; 1292 enum ttm_caching caching; 1293 1294 gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL); 1295 if (gtt == NULL) { 1296 return NULL; 1297 } 1298 gtt->gobj = &bo->base; 1299 1300 if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 1301 caching = ttm_write_combined; 1302 else 1303 caching = ttm_cached; 1304 1305 /* allocate space for the uninitialized page entries */ 1306 if (ttm_sg_tt_init(>t->ttm, bo, page_flags, caching)) { 1307 kfree(gtt); 1308 return NULL; 1309 } 1310 return >t->ttm.ttm; 1311 } 1312 1313 /** 1314 * amdgpu_ttm_tt_populate - Map GTT pages visible to the device 1315 * 1316 * Map the pages of a ttm_tt object to an address space visible 1317 * to the underlying device. 1318 */ 1319 static int amdgpu_ttm_tt_populate(struct ttm_bo_device *bdev, 1320 struct ttm_tt *ttm, 1321 struct ttm_operation_ctx *ctx) 1322 { 1323 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 1324 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1325 1326 /* user pages are bound by amdgpu_ttm_tt_pin_userptr() */ 1327 if (gtt && gtt->userptr) { 1328 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); 1329 if (!ttm->sg) 1330 return -ENOMEM; 1331 1332 ttm->page_flags |= TTM_PAGE_FLAG_SG; 1333 ttm_tt_set_populated(ttm); 1334 return 0; 1335 } 1336 1337 if (ttm->page_flags & TTM_PAGE_FLAG_SG) { 1338 if (!ttm->sg) { 1339 struct dma_buf_attachment *attach; 1340 struct sg_table *sgt; 1341 1342 attach = gtt->gobj->import_attach; 1343 sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); 1344 if (IS_ERR(sgt)) 1345 return PTR_ERR(sgt); 1346 1347 ttm->sg = sgt; 1348 } 1349 1350 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 1351 gtt->ttm.dma_address, 1352 ttm->num_pages); 1353 ttm_tt_set_populated(ttm); 1354 return 0; 1355 } 1356 1357 #ifdef CONFIG_SWIOTLB 1358 if (adev->need_swiotlb && swiotlb_nr_tbl()) { 1359 return ttm_dma_populate(>t->ttm, adev->dev, ctx); 1360 } 1361 #endif 1362 1363 /* fall back to generic helper to populate the page array 1364 * and map them to the device */ 1365 return ttm_populate_and_map_pages(adev->dev, >t->ttm, ctx); 1366 } 1367 1368 /** 1369 * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays 1370 * 1371 * Unmaps pages of a ttm_tt object from the device address space and 1372 * unpopulates the page array backing it. 1373 */ 1374 static void amdgpu_ttm_tt_unpopulate(struct ttm_bo_device *bdev, struct ttm_tt *ttm) 1375 { 1376 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1377 struct amdgpu_device *adev; 1378 1379 if (gtt && gtt->userptr) { 1380 amdgpu_ttm_tt_set_user_pages(ttm, NULL); 1381 kfree(ttm->sg); 1382 ttm->page_flags &= ~TTM_PAGE_FLAG_SG; 1383 return; 1384 } 1385 1386 if (ttm->sg && gtt->gobj->import_attach) { 1387 struct dma_buf_attachment *attach; 1388 1389 attach = gtt->gobj->import_attach; 1390 dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL); 1391 ttm->sg = NULL; 1392 return; 1393 } 1394 1395 if (ttm->page_flags & TTM_PAGE_FLAG_SG) 1396 return; 1397 1398 adev = amdgpu_ttm_adev(bdev); 1399 1400 #ifdef CONFIG_SWIOTLB 1401 if (adev->need_swiotlb && swiotlb_nr_tbl()) { 1402 ttm_dma_unpopulate(>t->ttm, adev->dev); 1403 return; 1404 } 1405 #endif 1406 1407 /* fall back to generic helper to unmap and unpopulate array */ 1408 ttm_unmap_and_unpopulate_pages(adev->dev, >t->ttm); 1409 } 1410 1411 /** 1412 * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current 1413 * task 1414 * 1415 * @bo: The ttm_buffer_object to bind this userptr to 1416 * @addr: The address in the current tasks VM space to use 1417 * @flags: Requirements of userptr object. 1418 * 1419 * Called by amdgpu_gem_userptr_ioctl() to bind userptr pages 1420 * to current task 1421 */ 1422 int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo, 1423 uint64_t addr, uint32_t flags) 1424 { 1425 struct amdgpu_ttm_tt *gtt; 1426 1427 if (!bo->ttm) { 1428 /* TODO: We want a separate TTM object type for userptrs */ 1429 bo->ttm = amdgpu_ttm_tt_create(bo, 0); 1430 if (bo->ttm == NULL) 1431 return -ENOMEM; 1432 } 1433 1434 gtt = (void*)bo->ttm; 1435 gtt->userptr = addr; 1436 gtt->userflags = flags; 1437 1438 if (gtt->usertask) 1439 put_task_struct(gtt->usertask); 1440 gtt->usertask = current->group_leader; 1441 get_task_struct(gtt->usertask); 1442 1443 return 0; 1444 } 1445 1446 /** 1447 * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object 1448 */ 1449 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm) 1450 { 1451 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1452 1453 if (gtt == NULL) 1454 return NULL; 1455 1456 if (gtt->usertask == NULL) 1457 return NULL; 1458 1459 return gtt->usertask->mm; 1460 } 1461 1462 /** 1463 * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an 1464 * address range for the current task. 1465 * 1466 */ 1467 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, 1468 unsigned long end) 1469 { 1470 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1471 unsigned long size; 1472 1473 if (gtt == NULL || !gtt->userptr) 1474 return false; 1475 1476 /* Return false if no part of the ttm_tt object lies within 1477 * the range 1478 */ 1479 size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE; 1480 if (gtt->userptr > end || gtt->userptr + size <= start) 1481 return false; 1482 1483 return true; 1484 } 1485 1486 /** 1487 * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr? 1488 */ 1489 bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm) 1490 { 1491 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1492 1493 if (gtt == NULL || !gtt->userptr) 1494 return false; 1495 1496 return true; 1497 } 1498 1499 /** 1500 * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only? 1501 */ 1502 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm) 1503 { 1504 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1505 1506 if (gtt == NULL) 1507 return false; 1508 1509 return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); 1510 } 1511 1512 /** 1513 * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object 1514 * 1515 * @ttm: The ttm_tt object to compute the flags for 1516 * @mem: The memory registry backing this ttm_tt object 1517 * 1518 * Figure out the flags to use for a VM PDE (Page Directory Entry). 1519 */ 1520 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem) 1521 { 1522 uint64_t flags = 0; 1523 1524 if (mem && mem->mem_type != TTM_PL_SYSTEM) 1525 flags |= AMDGPU_PTE_VALID; 1526 1527 if (mem && mem->mem_type == TTM_PL_TT) { 1528 flags |= AMDGPU_PTE_SYSTEM; 1529 1530 if (ttm->caching == ttm_cached) 1531 flags |= AMDGPU_PTE_SNOOPED; 1532 } 1533 1534 return flags; 1535 } 1536 1537 /** 1538 * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object 1539 * 1540 * @ttm: The ttm_tt object to compute the flags for 1541 * @mem: The memory registry backing this ttm_tt object 1542 1543 * Figure out the flags to use for a VM PTE (Page Table Entry). 1544 */ 1545 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm, 1546 struct ttm_resource *mem) 1547 { 1548 uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem); 1549 1550 flags |= adev->gart.gart_pte_flags; 1551 flags |= AMDGPU_PTE_READABLE; 1552 1553 if (!amdgpu_ttm_tt_is_readonly(ttm)) 1554 flags |= AMDGPU_PTE_WRITEABLE; 1555 1556 return flags; 1557 } 1558 1559 /** 1560 * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer 1561 * object. 1562 * 1563 * Return true if eviction is sensible. Called by ttm_mem_evict_first() on 1564 * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until 1565 * it can find space for a new object and by ttm_bo_force_list_clean() which is 1566 * used to clean out a memory space. 1567 */ 1568 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 1569 const struct ttm_place *place) 1570 { 1571 unsigned long num_pages = bo->mem.num_pages; 1572 struct drm_mm_node *node = bo->mem.mm_node; 1573 struct dma_resv_list *flist; 1574 struct dma_fence *f; 1575 int i; 1576 1577 if (bo->type == ttm_bo_type_kernel && 1578 !amdgpu_vm_evictable(ttm_to_amdgpu_bo(bo))) 1579 return false; 1580 1581 /* If bo is a KFD BO, check if the bo belongs to the current process. 1582 * If true, then return false as any KFD process needs all its BOs to 1583 * be resident to run successfully 1584 */ 1585 flist = dma_resv_get_list(bo->base.resv); 1586 if (flist) { 1587 for (i = 0; i < flist->shared_count; ++i) { 1588 f = rcu_dereference_protected(flist->shared[i], 1589 dma_resv_held(bo->base.resv)); 1590 if (amdkfd_fence_check_mm(f, current->mm)) 1591 return false; 1592 } 1593 } 1594 1595 switch (bo->mem.mem_type) { 1596 case TTM_PL_TT: 1597 if (amdgpu_bo_is_amdgpu_bo(bo) && 1598 amdgpu_bo_encrypted(ttm_to_amdgpu_bo(bo))) 1599 return false; 1600 return true; 1601 1602 case TTM_PL_VRAM: 1603 /* Check each drm MM node individually */ 1604 while (num_pages) { 1605 if (place->fpfn < (node->start + node->size) && 1606 !(place->lpfn && place->lpfn <= node->start)) 1607 return true; 1608 1609 num_pages -= node->size; 1610 ++node; 1611 } 1612 return false; 1613 1614 default: 1615 break; 1616 } 1617 1618 return ttm_bo_eviction_valuable(bo, place); 1619 } 1620 1621 /** 1622 * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object. 1623 * 1624 * @bo: The buffer object to read/write 1625 * @offset: Offset into buffer object 1626 * @buf: Secondary buffer to write/read from 1627 * @len: Length in bytes of access 1628 * @write: true if writing 1629 * 1630 * This is used to access VRAM that backs a buffer object via MMIO 1631 * access for debugging purposes. 1632 */ 1633 static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo, 1634 unsigned long offset, 1635 void *buf, int len, int write) 1636 { 1637 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 1638 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 1639 struct drm_mm_node *nodes; 1640 uint32_t value = 0; 1641 int ret = 0; 1642 uint64_t pos; 1643 unsigned long flags; 1644 1645 if (bo->mem.mem_type != TTM_PL_VRAM) 1646 return -EIO; 1647 1648 pos = offset; 1649 nodes = amdgpu_find_mm_node(&abo->tbo.mem, &pos); 1650 pos += (nodes->start << PAGE_SHIFT); 1651 1652 while (len && pos < adev->gmc.mc_vram_size) { 1653 uint64_t aligned_pos = pos & ~(uint64_t)3; 1654 uint64_t bytes = 4 - (pos & 3); 1655 uint32_t shift = (pos & 3) * 8; 1656 uint32_t mask = 0xffffffff << shift; 1657 1658 if (len < bytes) { 1659 mask &= 0xffffffff >> (bytes - len) * 8; 1660 bytes = len; 1661 } 1662 1663 if (mask != 0xffffffff) { 1664 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 1665 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000); 1666 WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31); 1667 if (!write || mask != 0xffffffff) 1668 value = RREG32_NO_KIQ(mmMM_DATA); 1669 if (write) { 1670 value &= ~mask; 1671 value |= (*(uint32_t *)buf << shift) & mask; 1672 WREG32_NO_KIQ(mmMM_DATA, value); 1673 } 1674 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 1675 if (!write) { 1676 value = (value & mask) >> shift; 1677 memcpy(buf, &value, bytes); 1678 } 1679 } else { 1680 bytes = (nodes->start + nodes->size) << PAGE_SHIFT; 1681 bytes = min(bytes - pos, (uint64_t)len & ~0x3ull); 1682 1683 amdgpu_device_vram_access(adev, pos, (uint32_t *)buf, 1684 bytes, write); 1685 } 1686 1687 ret += bytes; 1688 buf = (uint8_t *)buf + bytes; 1689 pos += bytes; 1690 len -= bytes; 1691 if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) { 1692 ++nodes; 1693 pos = (nodes->start << PAGE_SHIFT); 1694 } 1695 } 1696 1697 return ret; 1698 } 1699 1700 static struct ttm_bo_driver amdgpu_bo_driver = { 1701 .ttm_tt_create = &amdgpu_ttm_tt_create, 1702 .ttm_tt_populate = &amdgpu_ttm_tt_populate, 1703 .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate, 1704 .ttm_tt_bind = &amdgpu_ttm_backend_bind, 1705 .ttm_tt_unbind = &amdgpu_ttm_backend_unbind, 1706 .ttm_tt_destroy = &amdgpu_ttm_backend_destroy, 1707 .eviction_valuable = amdgpu_ttm_bo_eviction_valuable, 1708 .evict_flags = &amdgpu_evict_flags, 1709 .move = &amdgpu_bo_move, 1710 .verify_access = &amdgpu_verify_access, 1711 .move_notify = &amdgpu_bo_move_notify, 1712 .release_notify = &amdgpu_bo_release_notify, 1713 .io_mem_reserve = &amdgpu_ttm_io_mem_reserve, 1714 .io_mem_pfn = amdgpu_ttm_io_mem_pfn, 1715 .access_memory = &amdgpu_ttm_access_memory, 1716 .del_from_lru_notify = &amdgpu_vm_del_from_lru_notify 1717 }; 1718 1719 /* 1720 * Firmware Reservation functions 1721 */ 1722 /** 1723 * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram 1724 * 1725 * @adev: amdgpu_device pointer 1726 * 1727 * free fw reserved vram if it has been reserved. 1728 */ 1729 static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev) 1730 { 1731 amdgpu_bo_free_kernel(&adev->mman.fw_vram_usage_reserved_bo, 1732 NULL, &adev->mman.fw_vram_usage_va); 1733 } 1734 1735 /** 1736 * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw 1737 * 1738 * @adev: amdgpu_device pointer 1739 * 1740 * create bo vram reservation from fw. 1741 */ 1742 static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev) 1743 { 1744 uint64_t vram_size = adev->gmc.visible_vram_size; 1745 1746 adev->mman.fw_vram_usage_va = NULL; 1747 adev->mman.fw_vram_usage_reserved_bo = NULL; 1748 1749 if (adev->mman.fw_vram_usage_size == 0 || 1750 adev->mman.fw_vram_usage_size > vram_size) 1751 return 0; 1752 1753 return amdgpu_bo_create_kernel_at(adev, 1754 adev->mman.fw_vram_usage_start_offset, 1755 adev->mman.fw_vram_usage_size, 1756 AMDGPU_GEM_DOMAIN_VRAM, 1757 &adev->mman.fw_vram_usage_reserved_bo, 1758 &adev->mman.fw_vram_usage_va); 1759 } 1760 1761 /* 1762 * Memoy training reservation functions 1763 */ 1764 1765 /** 1766 * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram 1767 * 1768 * @adev: amdgpu_device pointer 1769 * 1770 * free memory training reserved vram if it has been reserved. 1771 */ 1772 static int amdgpu_ttm_training_reserve_vram_fini(struct amdgpu_device *adev) 1773 { 1774 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; 1775 1776 ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT; 1777 amdgpu_bo_free_kernel(&ctx->c2p_bo, NULL, NULL); 1778 ctx->c2p_bo = NULL; 1779 1780 return 0; 1781 } 1782 1783 static void amdgpu_ttm_training_data_block_init(struct amdgpu_device *adev) 1784 { 1785 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; 1786 1787 memset(ctx, 0, sizeof(*ctx)); 1788 1789 ctx->c2p_train_data_offset = 1790 ALIGN((adev->gmc.mc_vram_size - adev->mman.discovery_tmr_size - SZ_1M), SZ_1M); 1791 ctx->p2c_train_data_offset = 1792 (adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET); 1793 ctx->train_data_size = 1794 GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES; 1795 1796 DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n", 1797 ctx->train_data_size, 1798 ctx->p2c_train_data_offset, 1799 ctx->c2p_train_data_offset); 1800 } 1801 1802 /* 1803 * reserve TMR memory at the top of VRAM which holds 1804 * IP Discovery data and is protected by PSP. 1805 */ 1806 static int amdgpu_ttm_reserve_tmr(struct amdgpu_device *adev) 1807 { 1808 int ret; 1809 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; 1810 bool mem_train_support = false; 1811 1812 if (!amdgpu_sriov_vf(adev)) { 1813 ret = amdgpu_mem_train_support(adev); 1814 if (ret == 1) 1815 mem_train_support = true; 1816 else if (ret == -1) 1817 return -EINVAL; 1818 else 1819 DRM_DEBUG("memory training does not support!\n"); 1820 } 1821 1822 /* 1823 * Query reserved tmr size through atom firmwareinfo for Sienna_Cichlid and onwards for all 1824 * the use cases (IP discovery/G6 memory training/profiling/diagnostic data.etc) 1825 * 1826 * Otherwise, fallback to legacy approach to check and reserve tmr block for ip 1827 * discovery data and G6 memory training data respectively 1828 */ 1829 adev->mman.discovery_tmr_size = 1830 amdgpu_atomfirmware_get_fw_reserved_fb_size(adev); 1831 if (!adev->mman.discovery_tmr_size) 1832 adev->mman.discovery_tmr_size = DISCOVERY_TMR_OFFSET; 1833 1834 if (mem_train_support) { 1835 /* reserve vram for mem train according to TMR location */ 1836 amdgpu_ttm_training_data_block_init(adev); 1837 ret = amdgpu_bo_create_kernel_at(adev, 1838 ctx->c2p_train_data_offset, 1839 ctx->train_data_size, 1840 AMDGPU_GEM_DOMAIN_VRAM, 1841 &ctx->c2p_bo, 1842 NULL); 1843 if (ret) { 1844 DRM_ERROR("alloc c2p_bo failed(%d)!\n", ret); 1845 amdgpu_ttm_training_reserve_vram_fini(adev); 1846 return ret; 1847 } 1848 ctx->init = PSP_MEM_TRAIN_RESERVE_SUCCESS; 1849 } 1850 1851 ret = amdgpu_bo_create_kernel_at(adev, 1852 adev->gmc.real_vram_size - adev->mman.discovery_tmr_size, 1853 adev->mman.discovery_tmr_size, 1854 AMDGPU_GEM_DOMAIN_VRAM, 1855 &adev->mman.discovery_memory, 1856 NULL); 1857 if (ret) { 1858 DRM_ERROR("alloc tmr failed(%d)!\n", ret); 1859 amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); 1860 return ret; 1861 } 1862 1863 return 0; 1864 } 1865 1866 /** 1867 * amdgpu_ttm_init - Init the memory management (ttm) as well as various 1868 * gtt/vram related fields. 1869 * 1870 * This initializes all of the memory space pools that the TTM layer 1871 * will need such as the GTT space (system memory mapped to the device), 1872 * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which 1873 * can be mapped per VMID. 1874 */ 1875 int amdgpu_ttm_init(struct amdgpu_device *adev) 1876 { 1877 uint64_t gtt_size; 1878 int r; 1879 u64 vis_vram_limit; 1880 1881 mutex_init(&adev->mman.gtt_window_lock); 1882 1883 /* No others user of address space so set it to 0 */ 1884 r = ttm_bo_device_init(&adev->mman.bdev, 1885 &amdgpu_bo_driver, 1886 adev_to_drm(adev)->anon_inode->i_mapping, 1887 adev_to_drm(adev)->vma_offset_manager, 1888 dma_addressing_limited(adev->dev)); 1889 if (r) { 1890 DRM_ERROR("failed initializing buffer object driver(%d).\n", r); 1891 return r; 1892 } 1893 adev->mman.initialized = true; 1894 1895 /* We opt to avoid OOM on system pages allocations */ 1896 adev->mman.bdev.no_retry = true; 1897 1898 /* Initialize VRAM pool with all of VRAM divided into pages */ 1899 r = amdgpu_vram_mgr_init(adev); 1900 if (r) { 1901 DRM_ERROR("Failed initializing VRAM heap.\n"); 1902 return r; 1903 } 1904 1905 /* Reduce size of CPU-visible VRAM if requested */ 1906 vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024; 1907 if (amdgpu_vis_vram_limit > 0 && 1908 vis_vram_limit <= adev->gmc.visible_vram_size) 1909 adev->gmc.visible_vram_size = vis_vram_limit; 1910 1911 /* Change the size here instead of the init above so only lpfn is affected */ 1912 amdgpu_ttm_set_buffer_funcs_status(adev, false); 1913 #ifdef CONFIG_64BIT 1914 adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base, 1915 adev->gmc.visible_vram_size); 1916 #endif 1917 1918 /* 1919 *The reserved vram for firmware must be pinned to the specified 1920 *place on the VRAM, so reserve it early. 1921 */ 1922 r = amdgpu_ttm_fw_reserve_vram_init(adev); 1923 if (r) { 1924 return r; 1925 } 1926 1927 /* 1928 * only NAVI10 and onwards ASIC support for IP discovery. 1929 * If IP discovery enabled, a block of memory should be 1930 * reserved for IP discovey. 1931 */ 1932 if (adev->mman.discovery_bin) { 1933 r = amdgpu_ttm_reserve_tmr(adev); 1934 if (r) 1935 return r; 1936 } 1937 1938 /* allocate memory as required for VGA 1939 * This is used for VGA emulation and pre-OS scanout buffers to 1940 * avoid display artifacts while transitioning between pre-OS 1941 * and driver. */ 1942 r = amdgpu_bo_create_kernel_at(adev, 0, adev->mman.stolen_vga_size, 1943 AMDGPU_GEM_DOMAIN_VRAM, 1944 &adev->mman.stolen_vga_memory, 1945 NULL); 1946 if (r) 1947 return r; 1948 r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_vga_size, 1949 adev->mman.stolen_extended_size, 1950 AMDGPU_GEM_DOMAIN_VRAM, 1951 &adev->mman.stolen_extended_memory, 1952 NULL); 1953 if (r) 1954 return r; 1955 1956 DRM_INFO("amdgpu: %uM of VRAM memory ready\n", 1957 (unsigned) (adev->gmc.real_vram_size / (1024 * 1024))); 1958 1959 /* Compute GTT size, either bsaed on 3/4th the size of RAM size 1960 * or whatever the user passed on module init */ 1961 if (amdgpu_gtt_size == -1) { 1962 struct sysinfo si; 1963 1964 si_meminfo(&si); 1965 gtt_size = min(max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20), 1966 adev->gmc.mc_vram_size), 1967 ((uint64_t)si.totalram * si.mem_unit * 3/4)); 1968 } 1969 else 1970 gtt_size = (uint64_t)amdgpu_gtt_size << 20; 1971 1972 /* Initialize GTT memory pool */ 1973 r = amdgpu_gtt_mgr_init(adev, gtt_size); 1974 if (r) { 1975 DRM_ERROR("Failed initializing GTT heap.\n"); 1976 return r; 1977 } 1978 DRM_INFO("amdgpu: %uM of GTT memory ready.\n", 1979 (unsigned)(gtt_size / (1024 * 1024))); 1980 1981 /* Initialize various on-chip memory pools */ 1982 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GDS, adev->gds.gds_size); 1983 if (r) { 1984 DRM_ERROR("Failed initializing GDS heap.\n"); 1985 return r; 1986 } 1987 1988 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GWS, adev->gds.gws_size); 1989 if (r) { 1990 DRM_ERROR("Failed initializing gws heap.\n"); 1991 return r; 1992 } 1993 1994 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_OA, adev->gds.oa_size); 1995 if (r) { 1996 DRM_ERROR("Failed initializing oa heap.\n"); 1997 return r; 1998 } 1999 2000 return 0; 2001 } 2002 2003 /** 2004 * amdgpu_ttm_late_init - Handle any late initialization for amdgpu_ttm 2005 */ 2006 void amdgpu_ttm_late_init(struct amdgpu_device *adev) 2007 { 2008 /* return the VGA stolen memory (if any) back to VRAM */ 2009 if (!adev->mman.keep_stolen_vga_memory) 2010 amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL); 2011 amdgpu_bo_free_kernel(&adev->mman.stolen_extended_memory, NULL, NULL); 2012 } 2013 2014 /** 2015 * amdgpu_ttm_fini - De-initialize the TTM memory pools 2016 */ 2017 void amdgpu_ttm_fini(struct amdgpu_device *adev) 2018 { 2019 if (!adev->mman.initialized) 2020 return; 2021 2022 amdgpu_ttm_training_reserve_vram_fini(adev); 2023 /* return the stolen vga memory back to VRAM */ 2024 if (adev->mman.keep_stolen_vga_memory) 2025 amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL); 2026 /* return the IP Discovery TMR memory back to VRAM */ 2027 amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); 2028 amdgpu_ttm_fw_reserve_vram_fini(adev); 2029 2030 if (adev->mman.aper_base_kaddr) 2031 iounmap(adev->mman.aper_base_kaddr); 2032 adev->mman.aper_base_kaddr = NULL; 2033 2034 amdgpu_vram_mgr_fini(adev); 2035 amdgpu_gtt_mgr_fini(adev); 2036 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS); 2037 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GWS); 2038 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA); 2039 ttm_bo_device_release(&adev->mman.bdev); 2040 adev->mman.initialized = false; 2041 DRM_INFO("amdgpu: ttm finalized\n"); 2042 } 2043 2044 /** 2045 * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions 2046 * 2047 * @adev: amdgpu_device pointer 2048 * @enable: true when we can use buffer functions. 2049 * 2050 * Enable/disable use of buffer functions during suspend/resume. This should 2051 * only be called at bootup or when userspace isn't running. 2052 */ 2053 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable) 2054 { 2055 struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 2056 uint64_t size; 2057 int r; 2058 2059 if (!adev->mman.initialized || amdgpu_in_reset(adev) || 2060 adev->mman.buffer_funcs_enabled == enable) 2061 return; 2062 2063 if (enable) { 2064 struct amdgpu_ring *ring; 2065 struct drm_gpu_scheduler *sched; 2066 2067 ring = adev->mman.buffer_funcs_ring; 2068 sched = &ring->sched; 2069 r = drm_sched_entity_init(&adev->mman.entity, 2070 DRM_SCHED_PRIORITY_KERNEL, &sched, 2071 1, NULL); 2072 if (r) { 2073 DRM_ERROR("Failed setting up TTM BO move entity (%d)\n", 2074 r); 2075 return; 2076 } 2077 } else { 2078 drm_sched_entity_destroy(&adev->mman.entity); 2079 dma_fence_put(man->move); 2080 man->move = NULL; 2081 } 2082 2083 /* this just adjusts TTM size idea, which sets lpfn to the correct value */ 2084 if (enable) 2085 size = adev->gmc.real_vram_size; 2086 else 2087 size = adev->gmc.visible_vram_size; 2088 man->size = size >> PAGE_SHIFT; 2089 adev->mman.buffer_funcs_enabled = enable; 2090 } 2091 2092 static vm_fault_t amdgpu_ttm_fault(struct vm_fault *vmf) 2093 { 2094 struct ttm_buffer_object *bo = vmf->vma->vm_private_data; 2095 vm_fault_t ret; 2096 2097 ret = ttm_bo_vm_reserve(bo, vmf); 2098 if (ret) 2099 return ret; 2100 2101 ret = amdgpu_bo_fault_reserve_notify(bo); 2102 if (ret) 2103 goto unlock; 2104 2105 ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, 2106 TTM_BO_VM_NUM_PREFAULT, 1); 2107 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) 2108 return ret; 2109 2110 unlock: 2111 dma_resv_unlock(bo->base.resv); 2112 return ret; 2113 } 2114 2115 static struct vm_operations_struct amdgpu_ttm_vm_ops = { 2116 .fault = amdgpu_ttm_fault, 2117 .open = ttm_bo_vm_open, 2118 .close = ttm_bo_vm_close, 2119 .access = ttm_bo_vm_access 2120 }; 2121 2122 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma) 2123 { 2124 struct drm_file *file_priv = filp->private_data; 2125 struct amdgpu_device *adev = drm_to_adev(file_priv->minor->dev); 2126 int r; 2127 2128 r = ttm_bo_mmap(filp, vma, &adev->mman.bdev); 2129 if (unlikely(r != 0)) 2130 return r; 2131 2132 vma->vm_ops = &amdgpu_ttm_vm_ops; 2133 return 0; 2134 } 2135 2136 int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset, 2137 uint64_t dst_offset, uint32_t byte_count, 2138 struct dma_resv *resv, 2139 struct dma_fence **fence, bool direct_submit, 2140 bool vm_needs_flush, bool tmz) 2141 { 2142 enum amdgpu_ib_pool_type pool = direct_submit ? AMDGPU_IB_POOL_DIRECT : 2143 AMDGPU_IB_POOL_DELAYED; 2144 struct amdgpu_device *adev = ring->adev; 2145 struct amdgpu_job *job; 2146 2147 uint32_t max_bytes; 2148 unsigned num_loops, num_dw; 2149 unsigned i; 2150 int r; 2151 2152 if (direct_submit && !ring->sched.ready) { 2153 DRM_ERROR("Trying to move memory with ring turned off.\n"); 2154 return -EINVAL; 2155 } 2156 2157 max_bytes = adev->mman.buffer_funcs->copy_max_bytes; 2158 num_loops = DIV_ROUND_UP(byte_count, max_bytes); 2159 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8); 2160 2161 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, pool, &job); 2162 if (r) 2163 return r; 2164 2165 if (vm_needs_flush) { 2166 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo); 2167 job->vm_needs_flush = true; 2168 } 2169 if (resv) { 2170 r = amdgpu_sync_resv(adev, &job->sync, resv, 2171 AMDGPU_SYNC_ALWAYS, 2172 AMDGPU_FENCE_OWNER_UNDEFINED); 2173 if (r) { 2174 DRM_ERROR("sync failed (%d).\n", r); 2175 goto error_free; 2176 } 2177 } 2178 2179 for (i = 0; i < num_loops; i++) { 2180 uint32_t cur_size_in_bytes = min(byte_count, max_bytes); 2181 2182 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset, 2183 dst_offset, cur_size_in_bytes, tmz); 2184 2185 src_offset += cur_size_in_bytes; 2186 dst_offset += cur_size_in_bytes; 2187 byte_count -= cur_size_in_bytes; 2188 } 2189 2190 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 2191 WARN_ON(job->ibs[0].length_dw > num_dw); 2192 if (direct_submit) 2193 r = amdgpu_job_submit_direct(job, ring, fence); 2194 else 2195 r = amdgpu_job_submit(job, &adev->mman.entity, 2196 AMDGPU_FENCE_OWNER_UNDEFINED, fence); 2197 if (r) 2198 goto error_free; 2199 2200 return r; 2201 2202 error_free: 2203 amdgpu_job_free(job); 2204 DRM_ERROR("Error scheduling IBs (%d)\n", r); 2205 return r; 2206 } 2207 2208 int amdgpu_fill_buffer(struct amdgpu_bo *bo, 2209 uint32_t src_data, 2210 struct dma_resv *resv, 2211 struct dma_fence **fence) 2212 { 2213 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 2214 uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes; 2215 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; 2216 2217 struct drm_mm_node *mm_node; 2218 unsigned long num_pages; 2219 unsigned int num_loops, num_dw; 2220 2221 struct amdgpu_job *job; 2222 int r; 2223 2224 if (!adev->mman.buffer_funcs_enabled) { 2225 DRM_ERROR("Trying to clear memory with ring turned off.\n"); 2226 return -EINVAL; 2227 } 2228 2229 if (bo->tbo.mem.mem_type == TTM_PL_TT) { 2230 r = amdgpu_ttm_alloc_gart(&bo->tbo); 2231 if (r) 2232 return r; 2233 } 2234 2235 num_pages = bo->tbo.num_pages; 2236 mm_node = bo->tbo.mem.mm_node; 2237 num_loops = 0; 2238 while (num_pages) { 2239 uint64_t byte_count = mm_node->size << PAGE_SHIFT; 2240 2241 num_loops += DIV_ROUND_UP_ULL(byte_count, max_bytes); 2242 num_pages -= mm_node->size; 2243 ++mm_node; 2244 } 2245 num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw; 2246 2247 /* for IB padding */ 2248 num_dw += 64; 2249 2250 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, AMDGPU_IB_POOL_DELAYED, 2251 &job); 2252 if (r) 2253 return r; 2254 2255 if (resv) { 2256 r = amdgpu_sync_resv(adev, &job->sync, resv, 2257 AMDGPU_SYNC_ALWAYS, 2258 AMDGPU_FENCE_OWNER_UNDEFINED); 2259 if (r) { 2260 DRM_ERROR("sync failed (%d).\n", r); 2261 goto error_free; 2262 } 2263 } 2264 2265 num_pages = bo->tbo.num_pages; 2266 mm_node = bo->tbo.mem.mm_node; 2267 2268 while (num_pages) { 2269 uint64_t byte_count = mm_node->size << PAGE_SHIFT; 2270 uint64_t dst_addr; 2271 2272 dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem); 2273 while (byte_count) { 2274 uint32_t cur_size_in_bytes = min_t(uint64_t, byte_count, 2275 max_bytes); 2276 2277 amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, 2278 dst_addr, cur_size_in_bytes); 2279 2280 dst_addr += cur_size_in_bytes; 2281 byte_count -= cur_size_in_bytes; 2282 } 2283 2284 num_pages -= mm_node->size; 2285 ++mm_node; 2286 } 2287 2288 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 2289 WARN_ON(job->ibs[0].length_dw > num_dw); 2290 r = amdgpu_job_submit(job, &adev->mman.entity, 2291 AMDGPU_FENCE_OWNER_UNDEFINED, fence); 2292 if (r) 2293 goto error_free; 2294 2295 return 0; 2296 2297 error_free: 2298 amdgpu_job_free(job); 2299 return r; 2300 } 2301 2302 #if defined(CONFIG_DEBUG_FS) 2303 2304 static int amdgpu_mm_dump_table(struct seq_file *m, void *data) 2305 { 2306 struct drm_info_node *node = (struct drm_info_node *)m->private; 2307 unsigned ttm_pl = (uintptr_t)node->info_ent->data; 2308 struct drm_device *dev = node->minor->dev; 2309 struct amdgpu_device *adev = drm_to_adev(dev); 2310 struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, ttm_pl); 2311 struct drm_printer p = drm_seq_file_printer(m); 2312 2313 man->func->debug(man, &p); 2314 return 0; 2315 } 2316 2317 static const struct drm_info_list amdgpu_ttm_debugfs_list[] = { 2318 {"amdgpu_vram_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_VRAM}, 2319 {"amdgpu_gtt_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_TT}, 2320 {"amdgpu_gds_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GDS}, 2321 {"amdgpu_gws_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GWS}, 2322 {"amdgpu_oa_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_OA}, 2323 {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL}, 2324 #ifdef CONFIG_SWIOTLB 2325 {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL} 2326 #endif 2327 }; 2328 2329 /** 2330 * amdgpu_ttm_vram_read - Linear read access to VRAM 2331 * 2332 * Accesses VRAM via MMIO for debugging purposes. 2333 */ 2334 static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf, 2335 size_t size, loff_t *pos) 2336 { 2337 struct amdgpu_device *adev = file_inode(f)->i_private; 2338 ssize_t result = 0; 2339 2340 if (size & 0x3 || *pos & 0x3) 2341 return -EINVAL; 2342 2343 if (*pos >= adev->gmc.mc_vram_size) 2344 return -ENXIO; 2345 2346 size = min(size, (size_t)(adev->gmc.mc_vram_size - *pos)); 2347 while (size) { 2348 size_t bytes = min(size, AMDGPU_TTM_VRAM_MAX_DW_READ * 4); 2349 uint32_t value[AMDGPU_TTM_VRAM_MAX_DW_READ]; 2350 2351 amdgpu_device_vram_access(adev, *pos, value, bytes, false); 2352 if (copy_to_user(buf, value, bytes)) 2353 return -EFAULT; 2354 2355 result += bytes; 2356 buf += bytes; 2357 *pos += bytes; 2358 size -= bytes; 2359 } 2360 2361 return result; 2362 } 2363 2364 /** 2365 * amdgpu_ttm_vram_write - Linear write access to VRAM 2366 * 2367 * Accesses VRAM via MMIO for debugging purposes. 2368 */ 2369 static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf, 2370 size_t size, loff_t *pos) 2371 { 2372 struct amdgpu_device *adev = file_inode(f)->i_private; 2373 ssize_t result = 0; 2374 int r; 2375 2376 if (size & 0x3 || *pos & 0x3) 2377 return -EINVAL; 2378 2379 if (*pos >= adev->gmc.mc_vram_size) 2380 return -ENXIO; 2381 2382 while (size) { 2383 unsigned long flags; 2384 uint32_t value; 2385 2386 if (*pos >= adev->gmc.mc_vram_size) 2387 return result; 2388 2389 r = get_user(value, (uint32_t *)buf); 2390 if (r) 2391 return r; 2392 2393 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 2394 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000); 2395 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31); 2396 WREG32_NO_KIQ(mmMM_DATA, value); 2397 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 2398 2399 result += 4; 2400 buf += 4; 2401 *pos += 4; 2402 size -= 4; 2403 } 2404 2405 return result; 2406 } 2407 2408 static const struct file_operations amdgpu_ttm_vram_fops = { 2409 .owner = THIS_MODULE, 2410 .read = amdgpu_ttm_vram_read, 2411 .write = amdgpu_ttm_vram_write, 2412 .llseek = default_llseek, 2413 }; 2414 2415 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS 2416 2417 /** 2418 * amdgpu_ttm_gtt_read - Linear read access to GTT memory 2419 */ 2420 static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf, 2421 size_t size, loff_t *pos) 2422 { 2423 struct amdgpu_device *adev = file_inode(f)->i_private; 2424 ssize_t result = 0; 2425 int r; 2426 2427 while (size) { 2428 loff_t p = *pos / PAGE_SIZE; 2429 unsigned off = *pos & ~PAGE_MASK; 2430 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); 2431 struct page *page; 2432 void *ptr; 2433 2434 if (p >= adev->gart.num_cpu_pages) 2435 return result; 2436 2437 page = adev->gart.pages[p]; 2438 if (page) { 2439 ptr = kmap(page); 2440 ptr += off; 2441 2442 r = copy_to_user(buf, ptr, cur_size); 2443 kunmap(adev->gart.pages[p]); 2444 } else 2445 r = clear_user(buf, cur_size); 2446 2447 if (r) 2448 return -EFAULT; 2449 2450 result += cur_size; 2451 buf += cur_size; 2452 *pos += cur_size; 2453 size -= cur_size; 2454 } 2455 2456 return result; 2457 } 2458 2459 static const struct file_operations amdgpu_ttm_gtt_fops = { 2460 .owner = THIS_MODULE, 2461 .read = amdgpu_ttm_gtt_read, 2462 .llseek = default_llseek 2463 }; 2464 2465 #endif 2466 2467 /** 2468 * amdgpu_iomem_read - Virtual read access to GPU mapped memory 2469 * 2470 * This function is used to read memory that has been mapped to the 2471 * GPU and the known addresses are not physical addresses but instead 2472 * bus addresses (e.g., what you'd put in an IB or ring buffer). 2473 */ 2474 static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf, 2475 size_t size, loff_t *pos) 2476 { 2477 struct amdgpu_device *adev = file_inode(f)->i_private; 2478 struct iommu_domain *dom; 2479 ssize_t result = 0; 2480 int r; 2481 2482 /* retrieve the IOMMU domain if any for this device */ 2483 dom = iommu_get_domain_for_dev(adev->dev); 2484 2485 while (size) { 2486 phys_addr_t addr = *pos & PAGE_MASK; 2487 loff_t off = *pos & ~PAGE_MASK; 2488 size_t bytes = PAGE_SIZE - off; 2489 unsigned long pfn; 2490 struct page *p; 2491 void *ptr; 2492 2493 bytes = bytes < size ? bytes : size; 2494 2495 /* Translate the bus address to a physical address. If 2496 * the domain is NULL it means there is no IOMMU active 2497 * and the address translation is the identity 2498 */ 2499 addr = dom ? iommu_iova_to_phys(dom, addr) : addr; 2500 2501 pfn = addr >> PAGE_SHIFT; 2502 if (!pfn_valid(pfn)) 2503 return -EPERM; 2504 2505 p = pfn_to_page(pfn); 2506 if (p->mapping != adev->mman.bdev.dev_mapping) 2507 return -EPERM; 2508 2509 ptr = kmap(p); 2510 r = copy_to_user(buf, ptr + off, bytes); 2511 kunmap(p); 2512 if (r) 2513 return -EFAULT; 2514 2515 size -= bytes; 2516 *pos += bytes; 2517 result += bytes; 2518 } 2519 2520 return result; 2521 } 2522 2523 /** 2524 * amdgpu_iomem_write - Virtual write access to GPU mapped memory 2525 * 2526 * This function is used to write memory that has been mapped to the 2527 * GPU and the known addresses are not physical addresses but instead 2528 * bus addresses (e.g., what you'd put in an IB or ring buffer). 2529 */ 2530 static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf, 2531 size_t size, loff_t *pos) 2532 { 2533 struct amdgpu_device *adev = file_inode(f)->i_private; 2534 struct iommu_domain *dom; 2535 ssize_t result = 0; 2536 int r; 2537 2538 dom = iommu_get_domain_for_dev(adev->dev); 2539 2540 while (size) { 2541 phys_addr_t addr = *pos & PAGE_MASK; 2542 loff_t off = *pos & ~PAGE_MASK; 2543 size_t bytes = PAGE_SIZE - off; 2544 unsigned long pfn; 2545 struct page *p; 2546 void *ptr; 2547 2548 bytes = bytes < size ? bytes : size; 2549 2550 addr = dom ? iommu_iova_to_phys(dom, addr) : addr; 2551 2552 pfn = addr >> PAGE_SHIFT; 2553 if (!pfn_valid(pfn)) 2554 return -EPERM; 2555 2556 p = pfn_to_page(pfn); 2557 if (p->mapping != adev->mman.bdev.dev_mapping) 2558 return -EPERM; 2559 2560 ptr = kmap(p); 2561 r = copy_from_user(ptr + off, buf, bytes); 2562 kunmap(p); 2563 if (r) 2564 return -EFAULT; 2565 2566 size -= bytes; 2567 *pos += bytes; 2568 result += bytes; 2569 } 2570 2571 return result; 2572 } 2573 2574 static const struct file_operations amdgpu_ttm_iomem_fops = { 2575 .owner = THIS_MODULE, 2576 .read = amdgpu_iomem_read, 2577 .write = amdgpu_iomem_write, 2578 .llseek = default_llseek 2579 }; 2580 2581 static const struct { 2582 char *name; 2583 const struct file_operations *fops; 2584 int domain; 2585 } ttm_debugfs_entries[] = { 2586 { "amdgpu_vram", &amdgpu_ttm_vram_fops, TTM_PL_VRAM }, 2587 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS 2588 { "amdgpu_gtt", &amdgpu_ttm_gtt_fops, TTM_PL_TT }, 2589 #endif 2590 { "amdgpu_iomem", &amdgpu_ttm_iomem_fops, TTM_PL_SYSTEM }, 2591 }; 2592 2593 #endif 2594 2595 int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev) 2596 { 2597 #if defined(CONFIG_DEBUG_FS) 2598 unsigned count; 2599 2600 struct drm_minor *minor = adev_to_drm(adev)->primary; 2601 struct dentry *ent, *root = minor->debugfs_root; 2602 2603 for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) { 2604 ent = debugfs_create_file( 2605 ttm_debugfs_entries[count].name, 2606 S_IFREG | S_IRUGO, root, 2607 adev, 2608 ttm_debugfs_entries[count].fops); 2609 if (IS_ERR(ent)) 2610 return PTR_ERR(ent); 2611 if (ttm_debugfs_entries[count].domain == TTM_PL_VRAM) 2612 i_size_write(ent->d_inode, adev->gmc.mc_vram_size); 2613 else if (ttm_debugfs_entries[count].domain == TTM_PL_TT) 2614 i_size_write(ent->d_inode, adev->gmc.gart_size); 2615 adev->mman.debugfs_entries[count] = ent; 2616 } 2617 2618 count = ARRAY_SIZE(amdgpu_ttm_debugfs_list); 2619 2620 #ifdef CONFIG_SWIOTLB 2621 if (!(adev->need_swiotlb && swiotlb_nr_tbl())) 2622 --count; 2623 #endif 2624 2625 return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count); 2626 #else 2627 return 0; 2628 #endif 2629 } 2630