1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 #include <drm/ttm/ttm_bo_api.h> 33 #include <drm/ttm/ttm_bo_driver.h> 34 #include <drm/ttm/ttm_placement.h> 35 #include <drm/ttm/ttm_module.h> 36 #include <drm/ttm/ttm_page_alloc.h> 37 #include <drm/drmP.h> 38 #include <drm/amdgpu_drm.h> 39 #include <linux/seq_file.h> 40 #include <linux/slab.h> 41 #include <linux/swiotlb.h> 42 #include <linux/swap.h> 43 #include <linux/pagemap.h> 44 #include <linux/debugfs.h> 45 #include <linux/iommu.h> 46 #include "amdgpu.h" 47 #include "amdgpu_object.h" 48 #include "amdgpu_trace.h" 49 #include "amdgpu_amdkfd.h" 50 #include "amdgpu_sdma.h" 51 #include "bif/bif_4_1_d.h" 52 53 static int amdgpu_map_buffer(struct ttm_buffer_object *bo, 54 struct ttm_mem_reg *mem, unsigned num_pages, 55 uint64_t offset, unsigned window, 56 struct amdgpu_ring *ring, 57 uint64_t *addr); 58 59 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev); 60 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev); 61 62 static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) 63 { 64 return 0; 65 } 66 67 /** 68 * amdgpu_init_mem_type - Initialize a memory manager for a specific type of 69 * memory request. 70 * 71 * @bdev: The TTM BO device object (contains a reference to amdgpu_device) 72 * @type: The type of memory requested 73 * @man: The memory type manager for each domain 74 * 75 * This is called by ttm_bo_init_mm() when a buffer object is being 76 * initialized. 77 */ 78 static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, 79 struct ttm_mem_type_manager *man) 80 { 81 struct amdgpu_device *adev; 82 83 adev = amdgpu_ttm_adev(bdev); 84 85 switch (type) { 86 case TTM_PL_SYSTEM: 87 /* System memory */ 88 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 89 man->available_caching = TTM_PL_MASK_CACHING; 90 man->default_caching = TTM_PL_FLAG_CACHED; 91 break; 92 case TTM_PL_TT: 93 /* GTT memory */ 94 man->func = &amdgpu_gtt_mgr_func; 95 man->gpu_offset = adev->gmc.gart_start; 96 man->available_caching = TTM_PL_MASK_CACHING; 97 man->default_caching = TTM_PL_FLAG_CACHED; 98 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA; 99 break; 100 case TTM_PL_VRAM: 101 /* "On-card" video ram */ 102 man->func = &amdgpu_vram_mgr_func; 103 man->gpu_offset = adev->gmc.vram_start; 104 man->flags = TTM_MEMTYPE_FLAG_FIXED | 105 TTM_MEMTYPE_FLAG_MAPPABLE; 106 man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC; 107 man->default_caching = TTM_PL_FLAG_WC; 108 break; 109 case AMDGPU_PL_GDS: 110 case AMDGPU_PL_GWS: 111 case AMDGPU_PL_OA: 112 /* On-chip GDS memory*/ 113 man->func = &ttm_bo_manager_func; 114 man->gpu_offset = 0; 115 man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA; 116 man->available_caching = TTM_PL_FLAG_UNCACHED; 117 man->default_caching = TTM_PL_FLAG_UNCACHED; 118 break; 119 default: 120 DRM_ERROR("Unsupported memory type %u\n", (unsigned)type); 121 return -EINVAL; 122 } 123 return 0; 124 } 125 126 /** 127 * amdgpu_evict_flags - Compute placement flags 128 * 129 * @bo: The buffer object to evict 130 * @placement: Possible destination(s) for evicted BO 131 * 132 * Fill in placement data when ttm_bo_evict() is called 133 */ 134 static void amdgpu_evict_flags(struct ttm_buffer_object *bo, 135 struct ttm_placement *placement) 136 { 137 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 138 struct amdgpu_bo *abo; 139 static const struct ttm_place placements = { 140 .fpfn = 0, 141 .lpfn = 0, 142 .flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM 143 }; 144 145 /* Don't handle scatter gather BOs */ 146 if (bo->type == ttm_bo_type_sg) { 147 placement->num_placement = 0; 148 placement->num_busy_placement = 0; 149 return; 150 } 151 152 /* Object isn't an AMDGPU object so ignore */ 153 if (!amdgpu_bo_is_amdgpu_bo(bo)) { 154 placement->placement = &placements; 155 placement->busy_placement = &placements; 156 placement->num_placement = 1; 157 placement->num_busy_placement = 1; 158 return; 159 } 160 161 abo = ttm_to_amdgpu_bo(bo); 162 switch (bo->mem.mem_type) { 163 case AMDGPU_PL_GDS: 164 case AMDGPU_PL_GWS: 165 case AMDGPU_PL_OA: 166 placement->num_placement = 0; 167 placement->num_busy_placement = 0; 168 return; 169 170 case TTM_PL_VRAM: 171 if (!adev->mman.buffer_funcs_enabled) { 172 /* Move to system memory */ 173 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); 174 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 175 !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) && 176 amdgpu_bo_in_cpu_visible_vram(abo)) { 177 178 /* Try evicting to the CPU inaccessible part of VRAM 179 * first, but only set GTT as busy placement, so this 180 * BO will be evicted to GTT rather than causing other 181 * BOs to be evicted from VRAM 182 */ 183 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | 184 AMDGPU_GEM_DOMAIN_GTT); 185 abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; 186 abo->placements[0].lpfn = 0; 187 abo->placement.busy_placement = &abo->placements[1]; 188 abo->placement.num_busy_placement = 1; 189 } else { 190 /* Move to GTT memory */ 191 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT); 192 } 193 break; 194 case TTM_PL_TT: 195 default: 196 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); 197 break; 198 } 199 *placement = abo->placement; 200 } 201 202 /** 203 * amdgpu_verify_access - Verify access for a mmap call 204 * 205 * @bo: The buffer object to map 206 * @filp: The file pointer from the process performing the mmap 207 * 208 * This is called by ttm_bo_mmap() to verify whether a process 209 * has the right to mmap a BO to their process space. 210 */ 211 static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp) 212 { 213 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 214 215 /* 216 * Don't verify access for KFD BOs. They don't have a GEM 217 * object associated with them. 218 */ 219 if (abo->kfd_bo) 220 return 0; 221 222 if (amdgpu_ttm_tt_get_usermm(bo->ttm)) 223 return -EPERM; 224 return drm_vma_node_verify_access(&abo->gem_base.vma_node, 225 filp->private_data); 226 } 227 228 /** 229 * amdgpu_move_null - Register memory for a buffer object 230 * 231 * @bo: The bo to assign the memory to 232 * @new_mem: The memory to be assigned. 233 * 234 * Assign the memory from new_mem to the memory of the buffer object bo. 235 */ 236 static void amdgpu_move_null(struct ttm_buffer_object *bo, 237 struct ttm_mem_reg *new_mem) 238 { 239 struct ttm_mem_reg *old_mem = &bo->mem; 240 241 BUG_ON(old_mem->mm_node != NULL); 242 *old_mem = *new_mem; 243 new_mem->mm_node = NULL; 244 } 245 246 /** 247 * amdgpu_mm_node_addr - Compute the GPU relative offset of a GTT buffer. 248 * 249 * @bo: The bo to assign the memory to. 250 * @mm_node: Memory manager node for drm allocator. 251 * @mem: The region where the bo resides. 252 * 253 */ 254 static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo, 255 struct drm_mm_node *mm_node, 256 struct ttm_mem_reg *mem) 257 { 258 uint64_t addr = 0; 259 260 if (mm_node->start != AMDGPU_BO_INVALID_OFFSET) { 261 addr = mm_node->start << PAGE_SHIFT; 262 addr += bo->bdev->man[mem->mem_type].gpu_offset; 263 } 264 return addr; 265 } 266 267 /** 268 * amdgpu_find_mm_node - Helper function finds the drm_mm_node corresponding to 269 * @offset. It also modifies the offset to be within the drm_mm_node returned 270 * 271 * @mem: The region where the bo resides. 272 * @offset: The offset that drm_mm_node is used for finding. 273 * 274 */ 275 static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem, 276 unsigned long *offset) 277 { 278 struct drm_mm_node *mm_node = mem->mm_node; 279 280 while (*offset >= (mm_node->size << PAGE_SHIFT)) { 281 *offset -= (mm_node->size << PAGE_SHIFT); 282 ++mm_node; 283 } 284 return mm_node; 285 } 286 287 /** 288 * amdgpu_copy_ttm_mem_to_mem - Helper function for copy 289 * 290 * The function copies @size bytes from {src->mem + src->offset} to 291 * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a 292 * move and different for a BO to BO copy. 293 * 294 * @f: Returns the last fence if multiple jobs are submitted. 295 */ 296 int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, 297 struct amdgpu_copy_mem *src, 298 struct amdgpu_copy_mem *dst, 299 uint64_t size, 300 struct reservation_object *resv, 301 struct dma_fence **f) 302 { 303 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; 304 struct drm_mm_node *src_mm, *dst_mm; 305 uint64_t src_node_start, dst_node_start, src_node_size, 306 dst_node_size, src_page_offset, dst_page_offset; 307 struct dma_fence *fence = NULL; 308 int r = 0; 309 const uint64_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE * 310 AMDGPU_GPU_PAGE_SIZE); 311 312 if (!adev->mman.buffer_funcs_enabled) { 313 DRM_ERROR("Trying to move memory with ring turned off.\n"); 314 return -EINVAL; 315 } 316 317 src_mm = amdgpu_find_mm_node(src->mem, &src->offset); 318 src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem) + 319 src->offset; 320 src_node_size = (src_mm->size << PAGE_SHIFT) - src->offset; 321 src_page_offset = src_node_start & (PAGE_SIZE - 1); 322 323 dst_mm = amdgpu_find_mm_node(dst->mem, &dst->offset); 324 dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, dst->mem) + 325 dst->offset; 326 dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst->offset; 327 dst_page_offset = dst_node_start & (PAGE_SIZE - 1); 328 329 mutex_lock(&adev->mman.gtt_window_lock); 330 331 while (size) { 332 unsigned long cur_size; 333 uint64_t from = src_node_start, to = dst_node_start; 334 struct dma_fence *next; 335 336 /* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst 337 * begins at an offset, then adjust the size accordingly 338 */ 339 cur_size = min3(min(src_node_size, dst_node_size), size, 340 GTT_MAX_BYTES); 341 if (cur_size + src_page_offset > GTT_MAX_BYTES || 342 cur_size + dst_page_offset > GTT_MAX_BYTES) 343 cur_size -= max(src_page_offset, dst_page_offset); 344 345 /* Map only what needs to be accessed. Map src to window 0 and 346 * dst to window 1 347 */ 348 if (src->mem->start == AMDGPU_BO_INVALID_OFFSET) { 349 r = amdgpu_map_buffer(src->bo, src->mem, 350 PFN_UP(cur_size + src_page_offset), 351 src_node_start, 0, ring, 352 &from); 353 if (r) 354 goto error; 355 /* Adjust the offset because amdgpu_map_buffer returns 356 * start of mapped page 357 */ 358 from += src_page_offset; 359 } 360 361 if (dst->mem->start == AMDGPU_BO_INVALID_OFFSET) { 362 r = amdgpu_map_buffer(dst->bo, dst->mem, 363 PFN_UP(cur_size + dst_page_offset), 364 dst_node_start, 1, ring, 365 &to); 366 if (r) 367 goto error; 368 to += dst_page_offset; 369 } 370 371 r = amdgpu_copy_buffer(ring, from, to, cur_size, 372 resv, &next, false, true); 373 if (r) 374 goto error; 375 376 dma_fence_put(fence); 377 fence = next; 378 379 size -= cur_size; 380 if (!size) 381 break; 382 383 src_node_size -= cur_size; 384 if (!src_node_size) { 385 src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm, 386 src->mem); 387 src_node_size = (src_mm->size << PAGE_SHIFT); 388 } else { 389 src_node_start += cur_size; 390 src_page_offset = src_node_start & (PAGE_SIZE - 1); 391 } 392 dst_node_size -= cur_size; 393 if (!dst_node_size) { 394 dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm, 395 dst->mem); 396 dst_node_size = (dst_mm->size << PAGE_SHIFT); 397 } else { 398 dst_node_start += cur_size; 399 dst_page_offset = dst_node_start & (PAGE_SIZE - 1); 400 } 401 } 402 error: 403 mutex_unlock(&adev->mman.gtt_window_lock); 404 if (f) 405 *f = dma_fence_get(fence); 406 dma_fence_put(fence); 407 return r; 408 } 409 410 /** 411 * amdgpu_move_blit - Copy an entire buffer to another buffer 412 * 413 * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to 414 * help move buffers to and from VRAM. 415 */ 416 static int amdgpu_move_blit(struct ttm_buffer_object *bo, 417 bool evict, bool no_wait_gpu, 418 struct ttm_mem_reg *new_mem, 419 struct ttm_mem_reg *old_mem) 420 { 421 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 422 struct amdgpu_copy_mem src, dst; 423 struct dma_fence *fence = NULL; 424 int r; 425 426 src.bo = bo; 427 dst.bo = bo; 428 src.mem = old_mem; 429 dst.mem = new_mem; 430 src.offset = 0; 431 dst.offset = 0; 432 433 r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst, 434 new_mem->num_pages << PAGE_SHIFT, 435 bo->resv, &fence); 436 if (r) 437 goto error; 438 439 /* Always block for VM page tables before committing the new location */ 440 if (bo->type == ttm_bo_type_kernel) 441 r = ttm_bo_move_accel_cleanup(bo, fence, true, new_mem); 442 else 443 r = ttm_bo_pipeline_move(bo, fence, evict, new_mem); 444 dma_fence_put(fence); 445 return r; 446 447 error: 448 if (fence) 449 dma_fence_wait(fence, false); 450 dma_fence_put(fence); 451 return r; 452 } 453 454 /** 455 * amdgpu_move_vram_ram - Copy VRAM buffer to RAM buffer 456 * 457 * Called by amdgpu_bo_move(). 458 */ 459 static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict, 460 struct ttm_operation_ctx *ctx, 461 struct ttm_mem_reg *new_mem) 462 { 463 struct amdgpu_device *adev; 464 struct ttm_mem_reg *old_mem = &bo->mem; 465 struct ttm_mem_reg tmp_mem; 466 struct ttm_place placements; 467 struct ttm_placement placement; 468 int r; 469 470 adev = amdgpu_ttm_adev(bo->bdev); 471 472 /* create space/pages for new_mem in GTT space */ 473 tmp_mem = *new_mem; 474 tmp_mem.mm_node = NULL; 475 placement.num_placement = 1; 476 placement.placement = &placements; 477 placement.num_busy_placement = 1; 478 placement.busy_placement = &placements; 479 placements.fpfn = 0; 480 placements.lpfn = 0; 481 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 482 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx); 483 if (unlikely(r)) { 484 return r; 485 } 486 487 /* set caching flags */ 488 r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement); 489 if (unlikely(r)) { 490 goto out_cleanup; 491 } 492 493 /* Bind the memory to the GTT space */ 494 r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx); 495 if (unlikely(r)) { 496 goto out_cleanup; 497 } 498 499 /* blit VRAM to GTT */ 500 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, &tmp_mem, old_mem); 501 if (unlikely(r)) { 502 goto out_cleanup; 503 } 504 505 /* move BO (in tmp_mem) to new_mem */ 506 r = ttm_bo_move_ttm(bo, ctx, new_mem); 507 out_cleanup: 508 ttm_bo_mem_put(bo, &tmp_mem); 509 return r; 510 } 511 512 /** 513 * amdgpu_move_ram_vram - Copy buffer from RAM to VRAM 514 * 515 * Called by amdgpu_bo_move(). 516 */ 517 static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict, 518 struct ttm_operation_ctx *ctx, 519 struct ttm_mem_reg *new_mem) 520 { 521 struct amdgpu_device *adev; 522 struct ttm_mem_reg *old_mem = &bo->mem; 523 struct ttm_mem_reg tmp_mem; 524 struct ttm_placement placement; 525 struct ttm_place placements; 526 int r; 527 528 adev = amdgpu_ttm_adev(bo->bdev); 529 530 /* make space in GTT for old_mem buffer */ 531 tmp_mem = *new_mem; 532 tmp_mem.mm_node = NULL; 533 placement.num_placement = 1; 534 placement.placement = &placements; 535 placement.num_busy_placement = 1; 536 placement.busy_placement = &placements; 537 placements.fpfn = 0; 538 placements.lpfn = 0; 539 placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT; 540 r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx); 541 if (unlikely(r)) { 542 return r; 543 } 544 545 /* move/bind old memory to GTT space */ 546 r = ttm_bo_move_ttm(bo, ctx, &tmp_mem); 547 if (unlikely(r)) { 548 goto out_cleanup; 549 } 550 551 /* copy to VRAM */ 552 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, new_mem, old_mem); 553 if (unlikely(r)) { 554 goto out_cleanup; 555 } 556 out_cleanup: 557 ttm_bo_mem_put(bo, &tmp_mem); 558 return r; 559 } 560 561 /** 562 * amdgpu_bo_move - Move a buffer object to a new memory location 563 * 564 * Called by ttm_bo_handle_move_mem() 565 */ 566 static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, 567 struct ttm_operation_ctx *ctx, 568 struct ttm_mem_reg *new_mem) 569 { 570 struct amdgpu_device *adev; 571 struct amdgpu_bo *abo; 572 struct ttm_mem_reg *old_mem = &bo->mem; 573 int r; 574 575 /* Can't move a pinned BO */ 576 abo = ttm_to_amdgpu_bo(bo); 577 if (WARN_ON_ONCE(abo->pin_count > 0)) 578 return -EINVAL; 579 580 adev = amdgpu_ttm_adev(bo->bdev); 581 582 if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 583 amdgpu_move_null(bo, new_mem); 584 return 0; 585 } 586 if ((old_mem->mem_type == TTM_PL_TT && 587 new_mem->mem_type == TTM_PL_SYSTEM) || 588 (old_mem->mem_type == TTM_PL_SYSTEM && 589 new_mem->mem_type == TTM_PL_TT)) { 590 /* bind is enough */ 591 amdgpu_move_null(bo, new_mem); 592 return 0; 593 } 594 if (old_mem->mem_type == AMDGPU_PL_GDS || 595 old_mem->mem_type == AMDGPU_PL_GWS || 596 old_mem->mem_type == AMDGPU_PL_OA || 597 new_mem->mem_type == AMDGPU_PL_GDS || 598 new_mem->mem_type == AMDGPU_PL_GWS || 599 new_mem->mem_type == AMDGPU_PL_OA) { 600 /* Nothing to save here */ 601 amdgpu_move_null(bo, new_mem); 602 return 0; 603 } 604 605 if (!adev->mman.buffer_funcs_enabled) 606 goto memcpy; 607 608 if (old_mem->mem_type == TTM_PL_VRAM && 609 new_mem->mem_type == TTM_PL_SYSTEM) { 610 r = amdgpu_move_vram_ram(bo, evict, ctx, new_mem); 611 } else if (old_mem->mem_type == TTM_PL_SYSTEM && 612 new_mem->mem_type == TTM_PL_VRAM) { 613 r = amdgpu_move_ram_vram(bo, evict, ctx, new_mem); 614 } else { 615 r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu, 616 new_mem, old_mem); 617 } 618 619 if (r) { 620 memcpy: 621 r = ttm_bo_move_memcpy(bo, ctx, new_mem); 622 if (r) { 623 return r; 624 } 625 } 626 627 if (bo->type == ttm_bo_type_device && 628 new_mem->mem_type == TTM_PL_VRAM && 629 old_mem->mem_type != TTM_PL_VRAM) { 630 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU 631 * accesses the BO after it's moved. 632 */ 633 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 634 } 635 636 /* update statistics */ 637 atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved); 638 return 0; 639 } 640 641 /** 642 * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault 643 * 644 * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault() 645 */ 646 static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 647 { 648 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 649 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 650 struct drm_mm_node *mm_node = mem->mm_node; 651 652 mem->bus.addr = NULL; 653 mem->bus.offset = 0; 654 mem->bus.size = mem->num_pages << PAGE_SHIFT; 655 mem->bus.base = 0; 656 mem->bus.is_iomem = false; 657 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) 658 return -EINVAL; 659 switch (mem->mem_type) { 660 case TTM_PL_SYSTEM: 661 /* system memory */ 662 return 0; 663 case TTM_PL_TT: 664 break; 665 case TTM_PL_VRAM: 666 mem->bus.offset = mem->start << PAGE_SHIFT; 667 /* check if it's visible */ 668 if ((mem->bus.offset + mem->bus.size) > adev->gmc.visible_vram_size) 669 return -EINVAL; 670 /* Only physically contiguous buffers apply. In a contiguous 671 * buffer, size of the first mm_node would match the number of 672 * pages in ttm_mem_reg. 673 */ 674 if (adev->mman.aper_base_kaddr && 675 (mm_node->size == mem->num_pages)) 676 mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr + 677 mem->bus.offset; 678 679 mem->bus.base = adev->gmc.aper_base; 680 mem->bus.is_iomem = true; 681 break; 682 default: 683 return -EINVAL; 684 } 685 return 0; 686 } 687 688 static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 689 { 690 } 691 692 static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo, 693 unsigned long page_offset) 694 { 695 struct drm_mm_node *mm; 696 unsigned long offset = (page_offset << PAGE_SHIFT); 697 698 mm = amdgpu_find_mm_node(&bo->mem, &offset); 699 return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start + 700 (offset >> PAGE_SHIFT); 701 } 702 703 /* 704 * TTM backend functions. 705 */ 706 struct amdgpu_ttm_gup_task_list { 707 struct list_head list; 708 struct task_struct *task; 709 }; 710 711 struct amdgpu_ttm_tt { 712 struct ttm_dma_tt ttm; 713 u64 offset; 714 uint64_t userptr; 715 struct task_struct *usertask; 716 uint32_t userflags; 717 spinlock_t guptasklock; 718 struct list_head guptasks; 719 atomic_t mmu_invalidations; 720 uint32_t last_set_pages; 721 }; 722 723 /** 724 * amdgpu_ttm_tt_get_user_pages - Pin pages of memory pointed to by a USERPTR 725 * pointer to memory 726 * 727 * Called by amdgpu_gem_userptr_ioctl() and amdgpu_cs_parser_bos(). 728 * This provides a wrapper around the get_user_pages() call to provide 729 * device accessible pages that back user memory. 730 */ 731 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages) 732 { 733 struct amdgpu_ttm_tt *gtt = (void *)ttm; 734 struct mm_struct *mm = gtt->usertask->mm; 735 unsigned int flags = 0; 736 unsigned pinned = 0; 737 int r; 738 739 if (!mm) /* Happens during process shutdown */ 740 return -ESRCH; 741 742 if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY)) 743 flags |= FOLL_WRITE; 744 745 down_read(&mm->mmap_sem); 746 747 if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) { 748 /* 749 * check that we only use anonymous memory to prevent problems 750 * with writeback 751 */ 752 unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE; 753 struct vm_area_struct *vma; 754 755 vma = find_vma(mm, gtt->userptr); 756 if (!vma || vma->vm_file || vma->vm_end < end) { 757 up_read(&mm->mmap_sem); 758 return -EPERM; 759 } 760 } 761 762 /* loop enough times using contiguous pages of memory */ 763 do { 764 unsigned num_pages = ttm->num_pages - pinned; 765 uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE; 766 struct page **p = pages + pinned; 767 struct amdgpu_ttm_gup_task_list guptask; 768 769 guptask.task = current; 770 spin_lock(>t->guptasklock); 771 list_add(&guptask.list, >t->guptasks); 772 spin_unlock(>t->guptasklock); 773 774 if (mm == current->mm) 775 r = get_user_pages(userptr, num_pages, flags, p, NULL); 776 else 777 r = get_user_pages_remote(gtt->usertask, 778 mm, userptr, num_pages, 779 flags, p, NULL, NULL); 780 781 spin_lock(>t->guptasklock); 782 list_del(&guptask.list); 783 spin_unlock(>t->guptasklock); 784 785 if (r < 0) 786 goto release_pages; 787 788 pinned += r; 789 790 } while (pinned < ttm->num_pages); 791 792 up_read(&mm->mmap_sem); 793 return 0; 794 795 release_pages: 796 release_pages(pages, pinned); 797 up_read(&mm->mmap_sem); 798 return r; 799 } 800 801 /** 802 * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary. 803 * 804 * Called by amdgpu_cs_list_validate(). This creates the page list 805 * that backs user memory and will ultimately be mapped into the device 806 * address space. 807 */ 808 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages) 809 { 810 struct amdgpu_ttm_tt *gtt = (void *)ttm; 811 unsigned i; 812 813 gtt->last_set_pages = atomic_read(>t->mmu_invalidations); 814 for (i = 0; i < ttm->num_pages; ++i) { 815 if (ttm->pages[i]) 816 put_page(ttm->pages[i]); 817 818 ttm->pages[i] = pages ? pages[i] : NULL; 819 } 820 } 821 822 /** 823 * amdgpu_ttm_tt_mark_user_page - Mark pages as dirty 824 * 825 * Called while unpinning userptr pages 826 */ 827 void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm) 828 { 829 struct amdgpu_ttm_tt *gtt = (void *)ttm; 830 unsigned i; 831 832 for (i = 0; i < ttm->num_pages; ++i) { 833 struct page *page = ttm->pages[i]; 834 835 if (!page) 836 continue; 837 838 if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY)) 839 set_page_dirty(page); 840 841 mark_page_accessed(page); 842 } 843 } 844 845 /** 846 * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages 847 * 848 * Called by amdgpu_ttm_backend_bind() 849 **/ 850 static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm) 851 { 852 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev); 853 struct amdgpu_ttm_tt *gtt = (void *)ttm; 854 unsigned nents; 855 int r; 856 857 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); 858 enum dma_data_direction direction = write ? 859 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 860 861 /* Allocate an SG array and squash pages into it */ 862 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, 863 ttm->num_pages << PAGE_SHIFT, 864 GFP_KERNEL); 865 if (r) 866 goto release_sg; 867 868 /* Map SG to device */ 869 r = -ENOMEM; 870 nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction); 871 if (nents != ttm->sg->nents) 872 goto release_sg; 873 874 /* convert SG to linear array of pages and dma addresses */ 875 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 876 gtt->ttm.dma_address, ttm->num_pages); 877 878 return 0; 879 880 release_sg: 881 kfree(ttm->sg); 882 return r; 883 } 884 885 /** 886 * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages 887 */ 888 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm) 889 { 890 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev); 891 struct amdgpu_ttm_tt *gtt = (void *)ttm; 892 893 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); 894 enum dma_data_direction direction = write ? 895 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 896 897 /* double check that we don't free the table twice */ 898 if (!ttm->sg->sgl) 899 return; 900 901 /* unmap the pages mapped to the device */ 902 dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction); 903 904 /* mark the pages as dirty */ 905 amdgpu_ttm_tt_mark_user_pages(ttm); 906 907 sg_free_table(ttm->sg); 908 } 909 910 int amdgpu_ttm_gart_bind(struct amdgpu_device *adev, 911 struct ttm_buffer_object *tbo, 912 uint64_t flags) 913 { 914 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo); 915 struct ttm_tt *ttm = tbo->ttm; 916 struct amdgpu_ttm_tt *gtt = (void *)ttm; 917 int r; 918 919 if (abo->flags & AMDGPU_GEM_CREATE_MQD_GFX9) { 920 uint64_t page_idx = 1; 921 922 r = amdgpu_gart_bind(adev, gtt->offset, page_idx, 923 ttm->pages, gtt->ttm.dma_address, flags); 924 if (r) 925 goto gart_bind_fail; 926 927 /* Patch mtype of the second part BO */ 928 flags &= ~AMDGPU_PTE_MTYPE_MASK; 929 flags |= AMDGPU_PTE_MTYPE(AMDGPU_MTYPE_NC); 930 931 r = amdgpu_gart_bind(adev, 932 gtt->offset + (page_idx << PAGE_SHIFT), 933 ttm->num_pages - page_idx, 934 &ttm->pages[page_idx], 935 &(gtt->ttm.dma_address[page_idx]), flags); 936 } else { 937 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, 938 ttm->pages, gtt->ttm.dma_address, flags); 939 } 940 941 gart_bind_fail: 942 if (r) 943 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n", 944 ttm->num_pages, gtt->offset); 945 946 return r; 947 } 948 949 /** 950 * amdgpu_ttm_backend_bind - Bind GTT memory 951 * 952 * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem(). 953 * This handles binding GTT memory to the device address space. 954 */ 955 static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm, 956 struct ttm_mem_reg *bo_mem) 957 { 958 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev); 959 struct amdgpu_ttm_tt *gtt = (void*)ttm; 960 uint64_t flags; 961 int r = 0; 962 963 if (gtt->userptr) { 964 r = amdgpu_ttm_tt_pin_userptr(ttm); 965 if (r) { 966 DRM_ERROR("failed to pin userptr\n"); 967 return r; 968 } 969 } 970 if (!ttm->num_pages) { 971 WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n", 972 ttm->num_pages, bo_mem, ttm); 973 } 974 975 if (bo_mem->mem_type == AMDGPU_PL_GDS || 976 bo_mem->mem_type == AMDGPU_PL_GWS || 977 bo_mem->mem_type == AMDGPU_PL_OA) 978 return -EINVAL; 979 980 if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) { 981 gtt->offset = AMDGPU_BO_INVALID_OFFSET; 982 return 0; 983 } 984 985 /* compute PTE flags relevant to this BO memory */ 986 flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem); 987 988 /* bind pages into GART page tables */ 989 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT; 990 r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, 991 ttm->pages, gtt->ttm.dma_address, flags); 992 993 if (r) 994 DRM_ERROR("failed to bind %lu pages at 0x%08llX\n", 995 ttm->num_pages, gtt->offset); 996 return r; 997 } 998 999 /** 1000 * amdgpu_ttm_alloc_gart - Allocate GART memory for buffer object 1001 */ 1002 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo) 1003 { 1004 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1005 struct ttm_operation_ctx ctx = { false, false }; 1006 struct amdgpu_ttm_tt *gtt = (void*)bo->ttm; 1007 struct ttm_mem_reg tmp; 1008 struct ttm_placement placement; 1009 struct ttm_place placements; 1010 uint64_t addr, flags; 1011 int r; 1012 1013 if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET) 1014 return 0; 1015 1016 addr = amdgpu_gmc_agp_addr(bo); 1017 if (addr != AMDGPU_BO_INVALID_OFFSET) { 1018 bo->mem.start = addr >> PAGE_SHIFT; 1019 } else { 1020 1021 /* allocate GART space */ 1022 tmp = bo->mem; 1023 tmp.mm_node = NULL; 1024 placement.num_placement = 1; 1025 placement.placement = &placements; 1026 placement.num_busy_placement = 1; 1027 placement.busy_placement = &placements; 1028 placements.fpfn = 0; 1029 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT; 1030 placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) | 1031 TTM_PL_FLAG_TT; 1032 1033 r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx); 1034 if (unlikely(r)) 1035 return r; 1036 1037 /* compute PTE flags for this buffer object */ 1038 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp); 1039 1040 /* Bind pages */ 1041 gtt->offset = (u64)tmp.start << PAGE_SHIFT; 1042 r = amdgpu_ttm_gart_bind(adev, bo, flags); 1043 if (unlikely(r)) { 1044 ttm_bo_mem_put(bo, &tmp); 1045 return r; 1046 } 1047 1048 ttm_bo_mem_put(bo, &bo->mem); 1049 bo->mem = tmp; 1050 } 1051 1052 bo->offset = (bo->mem.start << PAGE_SHIFT) + 1053 bo->bdev->man[bo->mem.mem_type].gpu_offset; 1054 1055 return 0; 1056 } 1057 1058 /** 1059 * amdgpu_ttm_recover_gart - Rebind GTT pages 1060 * 1061 * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to 1062 * rebind GTT pages during a GPU reset. 1063 */ 1064 int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo) 1065 { 1066 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); 1067 uint64_t flags; 1068 int r; 1069 1070 if (!tbo->ttm) 1071 return 0; 1072 1073 flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem); 1074 r = amdgpu_ttm_gart_bind(adev, tbo, flags); 1075 1076 return r; 1077 } 1078 1079 /** 1080 * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages 1081 * 1082 * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and 1083 * ttm_tt_destroy(). 1084 */ 1085 static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm) 1086 { 1087 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev); 1088 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1089 int r; 1090 1091 /* if the pages have userptr pinning then clear that first */ 1092 if (gtt->userptr) 1093 amdgpu_ttm_tt_unpin_userptr(ttm); 1094 1095 if (gtt->offset == AMDGPU_BO_INVALID_OFFSET) 1096 return 0; 1097 1098 /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */ 1099 r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages); 1100 if (r) 1101 DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n", 1102 gtt->ttm.ttm.num_pages, gtt->offset); 1103 return r; 1104 } 1105 1106 static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm) 1107 { 1108 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1109 1110 if (gtt->usertask) 1111 put_task_struct(gtt->usertask); 1112 1113 ttm_dma_tt_fini(>t->ttm); 1114 kfree(gtt); 1115 } 1116 1117 static struct ttm_backend_func amdgpu_backend_func = { 1118 .bind = &amdgpu_ttm_backend_bind, 1119 .unbind = &amdgpu_ttm_backend_unbind, 1120 .destroy = &amdgpu_ttm_backend_destroy, 1121 }; 1122 1123 /** 1124 * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO 1125 * 1126 * @bo: The buffer object to create a GTT ttm_tt object around 1127 * 1128 * Called by ttm_tt_create(). 1129 */ 1130 static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo, 1131 uint32_t page_flags) 1132 { 1133 struct amdgpu_device *adev; 1134 struct amdgpu_ttm_tt *gtt; 1135 1136 adev = amdgpu_ttm_adev(bo->bdev); 1137 1138 gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL); 1139 if (gtt == NULL) { 1140 return NULL; 1141 } 1142 gtt->ttm.ttm.func = &amdgpu_backend_func; 1143 1144 /* allocate space for the uninitialized page entries */ 1145 if (ttm_sg_tt_init(>t->ttm, bo, page_flags)) { 1146 kfree(gtt); 1147 return NULL; 1148 } 1149 return >t->ttm.ttm; 1150 } 1151 1152 /** 1153 * amdgpu_ttm_tt_populate - Map GTT pages visible to the device 1154 * 1155 * Map the pages of a ttm_tt object to an address space visible 1156 * to the underlying device. 1157 */ 1158 static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm, 1159 struct ttm_operation_ctx *ctx) 1160 { 1161 struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev); 1162 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1163 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 1164 1165 /* user pages are bound by amdgpu_ttm_tt_pin_userptr() */ 1166 if (gtt && gtt->userptr) { 1167 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); 1168 if (!ttm->sg) 1169 return -ENOMEM; 1170 1171 ttm->page_flags |= TTM_PAGE_FLAG_SG; 1172 ttm->state = tt_unbound; 1173 return 0; 1174 } 1175 1176 if (slave && ttm->sg) { 1177 drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, 1178 gtt->ttm.dma_address, 1179 ttm->num_pages); 1180 ttm->state = tt_unbound; 1181 return 0; 1182 } 1183 1184 #ifdef CONFIG_SWIOTLB 1185 if (adev->need_swiotlb && swiotlb_nr_tbl()) { 1186 return ttm_dma_populate(>t->ttm, adev->dev, ctx); 1187 } 1188 #endif 1189 1190 /* fall back to generic helper to populate the page array 1191 * and map them to the device */ 1192 return ttm_populate_and_map_pages(adev->dev, >t->ttm, ctx); 1193 } 1194 1195 /** 1196 * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays 1197 * 1198 * Unmaps pages of a ttm_tt object from the device address space and 1199 * unpopulates the page array backing it. 1200 */ 1201 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm) 1202 { 1203 struct amdgpu_device *adev; 1204 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1205 bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG); 1206 1207 if (gtt && gtt->userptr) { 1208 amdgpu_ttm_tt_set_user_pages(ttm, NULL); 1209 kfree(ttm->sg); 1210 ttm->page_flags &= ~TTM_PAGE_FLAG_SG; 1211 return; 1212 } 1213 1214 if (slave) 1215 return; 1216 1217 adev = amdgpu_ttm_adev(ttm->bdev); 1218 1219 #ifdef CONFIG_SWIOTLB 1220 if (adev->need_swiotlb && swiotlb_nr_tbl()) { 1221 ttm_dma_unpopulate(>t->ttm, adev->dev); 1222 return; 1223 } 1224 #endif 1225 1226 /* fall back to generic helper to unmap and unpopulate array */ 1227 ttm_unmap_and_unpopulate_pages(adev->dev, >t->ttm); 1228 } 1229 1230 /** 1231 * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current 1232 * task 1233 * 1234 * @ttm: The ttm_tt object to bind this userptr object to 1235 * @addr: The address in the current tasks VM space to use 1236 * @flags: Requirements of userptr object. 1237 * 1238 * Called by amdgpu_gem_userptr_ioctl() to bind userptr pages 1239 * to current task 1240 */ 1241 int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr, 1242 uint32_t flags) 1243 { 1244 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1245 1246 if (gtt == NULL) 1247 return -EINVAL; 1248 1249 gtt->userptr = addr; 1250 gtt->userflags = flags; 1251 1252 if (gtt->usertask) 1253 put_task_struct(gtt->usertask); 1254 gtt->usertask = current->group_leader; 1255 get_task_struct(gtt->usertask); 1256 1257 spin_lock_init(>t->guptasklock); 1258 INIT_LIST_HEAD(>t->guptasks); 1259 atomic_set(>t->mmu_invalidations, 0); 1260 gtt->last_set_pages = 0; 1261 1262 return 0; 1263 } 1264 1265 /** 1266 * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object 1267 */ 1268 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm) 1269 { 1270 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1271 1272 if (gtt == NULL) 1273 return NULL; 1274 1275 if (gtt->usertask == NULL) 1276 return NULL; 1277 1278 return gtt->usertask->mm; 1279 } 1280 1281 /** 1282 * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an 1283 * address range for the current task. 1284 * 1285 */ 1286 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, 1287 unsigned long end) 1288 { 1289 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1290 struct amdgpu_ttm_gup_task_list *entry; 1291 unsigned long size; 1292 1293 if (gtt == NULL || !gtt->userptr) 1294 return false; 1295 1296 /* Return false if no part of the ttm_tt object lies within 1297 * the range 1298 */ 1299 size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE; 1300 if (gtt->userptr > end || gtt->userptr + size <= start) 1301 return false; 1302 1303 /* Search the lists of tasks that hold this mapping and see 1304 * if current is one of them. If it is return false. 1305 */ 1306 spin_lock(>t->guptasklock); 1307 list_for_each_entry(entry, >t->guptasks, list) { 1308 if (entry->task == current) { 1309 spin_unlock(>t->guptasklock); 1310 return false; 1311 } 1312 } 1313 spin_unlock(>t->guptasklock); 1314 1315 atomic_inc(>t->mmu_invalidations); 1316 1317 return true; 1318 } 1319 1320 /** 1321 * amdgpu_ttm_tt_userptr_invalidated - Has the ttm_tt object been invalidated? 1322 */ 1323 bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm, 1324 int *last_invalidated) 1325 { 1326 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1327 int prev_invalidated = *last_invalidated; 1328 1329 *last_invalidated = atomic_read(>t->mmu_invalidations); 1330 return prev_invalidated != *last_invalidated; 1331 } 1332 1333 /** 1334 * amdgpu_ttm_tt_userptr_needs_pages - Have the pages backing this ttm_tt object 1335 * been invalidated since the last time they've been set? 1336 */ 1337 bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm) 1338 { 1339 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1340 1341 if (gtt == NULL || !gtt->userptr) 1342 return false; 1343 1344 return atomic_read(>t->mmu_invalidations) != gtt->last_set_pages; 1345 } 1346 1347 /** 1348 * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only? 1349 */ 1350 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm) 1351 { 1352 struct amdgpu_ttm_tt *gtt = (void *)ttm; 1353 1354 if (gtt == NULL) 1355 return false; 1356 1357 return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); 1358 } 1359 1360 /** 1361 * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object 1362 * 1363 * @ttm: The ttm_tt object to compute the flags for 1364 * @mem: The memory registry backing this ttm_tt object 1365 * 1366 * Figure out the flags to use for a VM PDE (Page Directory Entry). 1367 */ 1368 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem) 1369 { 1370 uint64_t flags = 0; 1371 1372 if (mem && mem->mem_type != TTM_PL_SYSTEM) 1373 flags |= AMDGPU_PTE_VALID; 1374 1375 if (mem && mem->mem_type == TTM_PL_TT) { 1376 flags |= AMDGPU_PTE_SYSTEM; 1377 1378 if (ttm->caching_state == tt_cached) 1379 flags |= AMDGPU_PTE_SNOOPED; 1380 } 1381 1382 return flags; 1383 } 1384 1385 /** 1386 * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object 1387 * 1388 * @ttm: The ttm_tt object to compute the flags for 1389 * @mem: The memory registry backing this ttm_tt object 1390 1391 * Figure out the flags to use for a VM PTE (Page Table Entry). 1392 */ 1393 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm, 1394 struct ttm_mem_reg *mem) 1395 { 1396 uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem); 1397 1398 flags |= adev->gart.gart_pte_flags; 1399 flags |= AMDGPU_PTE_READABLE; 1400 1401 if (!amdgpu_ttm_tt_is_readonly(ttm)) 1402 flags |= AMDGPU_PTE_WRITEABLE; 1403 1404 return flags; 1405 } 1406 1407 /** 1408 * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer 1409 * object. 1410 * 1411 * Return true if eviction is sensible. Called by ttm_mem_evict_first() on 1412 * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until 1413 * it can find space for a new object and by ttm_bo_force_list_clean() which is 1414 * used to clean out a memory space. 1415 */ 1416 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 1417 const struct ttm_place *place) 1418 { 1419 unsigned long num_pages = bo->mem.num_pages; 1420 struct drm_mm_node *node = bo->mem.mm_node; 1421 struct reservation_object_list *flist; 1422 struct dma_fence *f; 1423 int i; 1424 1425 /* Don't evict VM page tables while they are busy, otherwise we can't 1426 * cleanly handle page faults. 1427 */ 1428 if (bo->type == ttm_bo_type_kernel && 1429 !reservation_object_test_signaled_rcu(bo->resv, true)) 1430 return false; 1431 1432 /* If bo is a KFD BO, check if the bo belongs to the current process. 1433 * If true, then return false as any KFD process needs all its BOs to 1434 * be resident to run successfully 1435 */ 1436 flist = reservation_object_get_list(bo->resv); 1437 if (flist) { 1438 for (i = 0; i < flist->shared_count; ++i) { 1439 f = rcu_dereference_protected(flist->shared[i], 1440 reservation_object_held(bo->resv)); 1441 if (amdkfd_fence_check_mm(f, current->mm)) 1442 return false; 1443 } 1444 } 1445 1446 switch (bo->mem.mem_type) { 1447 case TTM_PL_TT: 1448 return true; 1449 1450 case TTM_PL_VRAM: 1451 /* Check each drm MM node individually */ 1452 while (num_pages) { 1453 if (place->fpfn < (node->start + node->size) && 1454 !(place->lpfn && place->lpfn <= node->start)) 1455 return true; 1456 1457 num_pages -= node->size; 1458 ++node; 1459 } 1460 return false; 1461 1462 default: 1463 break; 1464 } 1465 1466 return ttm_bo_eviction_valuable(bo, place); 1467 } 1468 1469 /** 1470 * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object. 1471 * 1472 * @bo: The buffer object to read/write 1473 * @offset: Offset into buffer object 1474 * @buf: Secondary buffer to write/read from 1475 * @len: Length in bytes of access 1476 * @write: true if writing 1477 * 1478 * This is used to access VRAM that backs a buffer object via MMIO 1479 * access for debugging purposes. 1480 */ 1481 static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo, 1482 unsigned long offset, 1483 void *buf, int len, int write) 1484 { 1485 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 1486 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 1487 struct drm_mm_node *nodes; 1488 uint32_t value = 0; 1489 int ret = 0; 1490 uint64_t pos; 1491 unsigned long flags; 1492 1493 if (bo->mem.mem_type != TTM_PL_VRAM) 1494 return -EIO; 1495 1496 nodes = amdgpu_find_mm_node(&abo->tbo.mem, &offset); 1497 pos = (nodes->start << PAGE_SHIFT) + offset; 1498 1499 while (len && pos < adev->gmc.mc_vram_size) { 1500 uint64_t aligned_pos = pos & ~(uint64_t)3; 1501 uint32_t bytes = 4 - (pos & 3); 1502 uint32_t shift = (pos & 3) * 8; 1503 uint32_t mask = 0xffffffff << shift; 1504 1505 if (len < bytes) { 1506 mask &= 0xffffffff >> (bytes - len) * 8; 1507 bytes = len; 1508 } 1509 1510 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 1511 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000); 1512 WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31); 1513 if (!write || mask != 0xffffffff) 1514 value = RREG32_NO_KIQ(mmMM_DATA); 1515 if (write) { 1516 value &= ~mask; 1517 value |= (*(uint32_t *)buf << shift) & mask; 1518 WREG32_NO_KIQ(mmMM_DATA, value); 1519 } 1520 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 1521 if (!write) { 1522 value = (value & mask) >> shift; 1523 memcpy(buf, &value, bytes); 1524 } 1525 1526 ret += bytes; 1527 buf = (uint8_t *)buf + bytes; 1528 pos += bytes; 1529 len -= bytes; 1530 if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) { 1531 ++nodes; 1532 pos = (nodes->start << PAGE_SHIFT); 1533 } 1534 } 1535 1536 return ret; 1537 } 1538 1539 static struct ttm_bo_driver amdgpu_bo_driver = { 1540 .ttm_tt_create = &amdgpu_ttm_tt_create, 1541 .ttm_tt_populate = &amdgpu_ttm_tt_populate, 1542 .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate, 1543 .invalidate_caches = &amdgpu_invalidate_caches, 1544 .init_mem_type = &amdgpu_init_mem_type, 1545 .eviction_valuable = amdgpu_ttm_bo_eviction_valuable, 1546 .evict_flags = &amdgpu_evict_flags, 1547 .move = &amdgpu_bo_move, 1548 .verify_access = &amdgpu_verify_access, 1549 .move_notify = &amdgpu_bo_move_notify, 1550 .fault_reserve_notify = &amdgpu_bo_fault_reserve_notify, 1551 .io_mem_reserve = &amdgpu_ttm_io_mem_reserve, 1552 .io_mem_free = &amdgpu_ttm_io_mem_free, 1553 .io_mem_pfn = amdgpu_ttm_io_mem_pfn, 1554 .access_memory = &amdgpu_ttm_access_memory, 1555 .del_from_lru_notify = &amdgpu_vm_del_from_lru_notify 1556 }; 1557 1558 /* 1559 * Firmware Reservation functions 1560 */ 1561 /** 1562 * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram 1563 * 1564 * @adev: amdgpu_device pointer 1565 * 1566 * free fw reserved vram if it has been reserved. 1567 */ 1568 static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev) 1569 { 1570 amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo, 1571 NULL, &adev->fw_vram_usage.va); 1572 } 1573 1574 /** 1575 * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw 1576 * 1577 * @adev: amdgpu_device pointer 1578 * 1579 * create bo vram reservation from fw. 1580 */ 1581 static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev) 1582 { 1583 struct ttm_operation_ctx ctx = { false, false }; 1584 struct amdgpu_bo_param bp; 1585 int r = 0; 1586 int i; 1587 u64 vram_size = adev->gmc.visible_vram_size; 1588 u64 offset = adev->fw_vram_usage.start_offset; 1589 u64 size = adev->fw_vram_usage.size; 1590 struct amdgpu_bo *bo; 1591 1592 memset(&bp, 0, sizeof(bp)); 1593 bp.size = adev->fw_vram_usage.size; 1594 bp.byte_align = PAGE_SIZE; 1595 bp.domain = AMDGPU_GEM_DOMAIN_VRAM; 1596 bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | 1597 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 1598 bp.type = ttm_bo_type_kernel; 1599 bp.resv = NULL; 1600 adev->fw_vram_usage.va = NULL; 1601 adev->fw_vram_usage.reserved_bo = NULL; 1602 1603 if (adev->fw_vram_usage.size > 0 && 1604 adev->fw_vram_usage.size <= vram_size) { 1605 1606 r = amdgpu_bo_create(adev, &bp, 1607 &adev->fw_vram_usage.reserved_bo); 1608 if (r) 1609 goto error_create; 1610 1611 r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false); 1612 if (r) 1613 goto error_reserve; 1614 1615 /* remove the original mem node and create a new one at the 1616 * request position 1617 */ 1618 bo = adev->fw_vram_usage.reserved_bo; 1619 offset = ALIGN(offset, PAGE_SIZE); 1620 for (i = 0; i < bo->placement.num_placement; ++i) { 1621 bo->placements[i].fpfn = offset >> PAGE_SHIFT; 1622 bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT; 1623 } 1624 1625 ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem); 1626 r = ttm_bo_mem_space(&bo->tbo, &bo->placement, 1627 &bo->tbo.mem, &ctx); 1628 if (r) 1629 goto error_pin; 1630 1631 r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo, 1632 AMDGPU_GEM_DOMAIN_VRAM, 1633 adev->fw_vram_usage.start_offset, 1634 (adev->fw_vram_usage.start_offset + 1635 adev->fw_vram_usage.size)); 1636 if (r) 1637 goto error_pin; 1638 r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo, 1639 &adev->fw_vram_usage.va); 1640 if (r) 1641 goto error_kmap; 1642 1643 amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo); 1644 } 1645 return r; 1646 1647 error_kmap: 1648 amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo); 1649 error_pin: 1650 amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo); 1651 error_reserve: 1652 amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo); 1653 error_create: 1654 adev->fw_vram_usage.va = NULL; 1655 adev->fw_vram_usage.reserved_bo = NULL; 1656 return r; 1657 } 1658 /** 1659 * amdgpu_ttm_init - Init the memory management (ttm) as well as various 1660 * gtt/vram related fields. 1661 * 1662 * This initializes all of the memory space pools that the TTM layer 1663 * will need such as the GTT space (system memory mapped to the device), 1664 * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which 1665 * can be mapped per VMID. 1666 */ 1667 int amdgpu_ttm_init(struct amdgpu_device *adev) 1668 { 1669 uint64_t gtt_size; 1670 int r; 1671 u64 vis_vram_limit; 1672 1673 mutex_init(&adev->mman.gtt_window_lock); 1674 1675 /* No others user of address space so set it to 0 */ 1676 r = ttm_bo_device_init(&adev->mman.bdev, 1677 &amdgpu_bo_driver, 1678 adev->ddev->anon_inode->i_mapping, 1679 adev->need_dma32); 1680 if (r) { 1681 DRM_ERROR("failed initializing buffer object driver(%d).\n", r); 1682 return r; 1683 } 1684 adev->mman.initialized = true; 1685 1686 /* We opt to avoid OOM on system pages allocations */ 1687 adev->mman.bdev.no_retry = true; 1688 1689 /* Initialize VRAM pool with all of VRAM divided into pages */ 1690 r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM, 1691 adev->gmc.real_vram_size >> PAGE_SHIFT); 1692 if (r) { 1693 DRM_ERROR("Failed initializing VRAM heap.\n"); 1694 return r; 1695 } 1696 1697 /* Reduce size of CPU-visible VRAM if requested */ 1698 vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024; 1699 if (amdgpu_vis_vram_limit > 0 && 1700 vis_vram_limit <= adev->gmc.visible_vram_size) 1701 adev->gmc.visible_vram_size = vis_vram_limit; 1702 1703 /* Change the size here instead of the init above so only lpfn is affected */ 1704 amdgpu_ttm_set_buffer_funcs_status(adev, false); 1705 #ifdef CONFIG_64BIT 1706 adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base, 1707 adev->gmc.visible_vram_size); 1708 #endif 1709 1710 /* 1711 *The reserved vram for firmware must be pinned to the specified 1712 *place on the VRAM, so reserve it early. 1713 */ 1714 r = amdgpu_ttm_fw_reserve_vram_init(adev); 1715 if (r) { 1716 return r; 1717 } 1718 1719 /* allocate memory as required for VGA 1720 * This is used for VGA emulation and pre-OS scanout buffers to 1721 * avoid display artifacts while transitioning between pre-OS 1722 * and driver. */ 1723 r = amdgpu_bo_create_kernel(adev, adev->gmc.stolen_size, PAGE_SIZE, 1724 AMDGPU_GEM_DOMAIN_VRAM, 1725 &adev->stolen_vga_memory, 1726 NULL, NULL); 1727 if (r) 1728 return r; 1729 DRM_INFO("amdgpu: %uM of VRAM memory ready\n", 1730 (unsigned) (adev->gmc.real_vram_size / (1024 * 1024))); 1731 1732 /* Compute GTT size, either bsaed on 3/4th the size of RAM size 1733 * or whatever the user passed on module init */ 1734 if (amdgpu_gtt_size == -1) { 1735 struct sysinfo si; 1736 1737 si_meminfo(&si); 1738 gtt_size = min(max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20), 1739 adev->gmc.mc_vram_size), 1740 ((uint64_t)si.totalram * si.mem_unit * 3/4)); 1741 } 1742 else 1743 gtt_size = (uint64_t)amdgpu_gtt_size << 20; 1744 1745 /* Initialize GTT memory pool */ 1746 r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT); 1747 if (r) { 1748 DRM_ERROR("Failed initializing GTT heap.\n"); 1749 return r; 1750 } 1751 DRM_INFO("amdgpu: %uM of GTT memory ready.\n", 1752 (unsigned)(gtt_size / (1024 * 1024))); 1753 1754 /* Initialize various on-chip memory pools */ 1755 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS, 1756 adev->gds.mem.total_size); 1757 if (r) { 1758 DRM_ERROR("Failed initializing GDS heap.\n"); 1759 return r; 1760 } 1761 1762 r = amdgpu_bo_create_kernel(adev, adev->gds.mem.gfx_partition_size, 1763 4, AMDGPU_GEM_DOMAIN_GDS, 1764 &adev->gds.gds_gfx_bo, NULL, NULL); 1765 if (r) 1766 return r; 1767 1768 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS, 1769 adev->gds.gws.total_size); 1770 if (r) { 1771 DRM_ERROR("Failed initializing gws heap.\n"); 1772 return r; 1773 } 1774 1775 r = amdgpu_bo_create_kernel(adev, adev->gds.gws.gfx_partition_size, 1776 1, AMDGPU_GEM_DOMAIN_GWS, 1777 &adev->gds.gws_gfx_bo, NULL, NULL); 1778 if (r) 1779 return r; 1780 1781 r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA, 1782 adev->gds.oa.total_size); 1783 if (r) { 1784 DRM_ERROR("Failed initializing oa heap.\n"); 1785 return r; 1786 } 1787 1788 r = amdgpu_bo_create_kernel(adev, adev->gds.oa.gfx_partition_size, 1789 1, AMDGPU_GEM_DOMAIN_OA, 1790 &adev->gds.oa_gfx_bo, NULL, NULL); 1791 if (r) 1792 return r; 1793 1794 /* Register debugfs entries for amdgpu_ttm */ 1795 r = amdgpu_ttm_debugfs_init(adev); 1796 if (r) { 1797 DRM_ERROR("Failed to init debugfs\n"); 1798 return r; 1799 } 1800 return 0; 1801 } 1802 1803 /** 1804 * amdgpu_ttm_late_init - Handle any late initialization for amdgpu_ttm 1805 */ 1806 void amdgpu_ttm_late_init(struct amdgpu_device *adev) 1807 { 1808 /* return the VGA stolen memory (if any) back to VRAM */ 1809 amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL); 1810 } 1811 1812 /** 1813 * amdgpu_ttm_fini - De-initialize the TTM memory pools 1814 */ 1815 void amdgpu_ttm_fini(struct amdgpu_device *adev) 1816 { 1817 if (!adev->mman.initialized) 1818 return; 1819 1820 amdgpu_ttm_debugfs_fini(adev); 1821 amdgpu_ttm_fw_reserve_vram_fini(adev); 1822 if (adev->mman.aper_base_kaddr) 1823 iounmap(adev->mman.aper_base_kaddr); 1824 adev->mman.aper_base_kaddr = NULL; 1825 1826 ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM); 1827 ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT); 1828 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS); 1829 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS); 1830 ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA); 1831 ttm_bo_device_release(&adev->mman.bdev); 1832 adev->mman.initialized = false; 1833 DRM_INFO("amdgpu: ttm finalized\n"); 1834 } 1835 1836 /** 1837 * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions 1838 * 1839 * @adev: amdgpu_device pointer 1840 * @enable: true when we can use buffer functions. 1841 * 1842 * Enable/disable use of buffer functions during suspend/resume. This should 1843 * only be called at bootup or when userspace isn't running. 1844 */ 1845 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable) 1846 { 1847 struct ttm_mem_type_manager *man = &adev->mman.bdev.man[TTM_PL_VRAM]; 1848 uint64_t size; 1849 int r; 1850 1851 if (!adev->mman.initialized || adev->in_gpu_reset || 1852 adev->mman.buffer_funcs_enabled == enable) 1853 return; 1854 1855 if (enable) { 1856 struct amdgpu_ring *ring; 1857 struct drm_sched_rq *rq; 1858 1859 ring = adev->mman.buffer_funcs_ring; 1860 rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL]; 1861 r = drm_sched_entity_init(&adev->mman.entity, &rq, 1, NULL); 1862 if (r) { 1863 DRM_ERROR("Failed setting up TTM BO move entity (%d)\n", 1864 r); 1865 return; 1866 } 1867 } else { 1868 drm_sched_entity_destroy(&adev->mman.entity); 1869 dma_fence_put(man->move); 1870 man->move = NULL; 1871 } 1872 1873 /* this just adjusts TTM size idea, which sets lpfn to the correct value */ 1874 if (enable) 1875 size = adev->gmc.real_vram_size; 1876 else 1877 size = adev->gmc.visible_vram_size; 1878 man->size = size >> PAGE_SHIFT; 1879 adev->mman.buffer_funcs_enabled = enable; 1880 } 1881 1882 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma) 1883 { 1884 struct drm_file *file_priv = filp->private_data; 1885 struct amdgpu_device *adev = file_priv->minor->dev->dev_private; 1886 1887 if (adev == NULL) 1888 return -EINVAL; 1889 1890 return ttm_bo_mmap(filp, vma, &adev->mman.bdev); 1891 } 1892 1893 static int amdgpu_map_buffer(struct ttm_buffer_object *bo, 1894 struct ttm_mem_reg *mem, unsigned num_pages, 1895 uint64_t offset, unsigned window, 1896 struct amdgpu_ring *ring, 1897 uint64_t *addr) 1898 { 1899 struct amdgpu_ttm_tt *gtt = (void *)bo->ttm; 1900 struct amdgpu_device *adev = ring->adev; 1901 struct ttm_tt *ttm = bo->ttm; 1902 struct amdgpu_job *job; 1903 unsigned num_dw, num_bytes; 1904 dma_addr_t *dma_address; 1905 struct dma_fence *fence; 1906 uint64_t src_addr, dst_addr; 1907 uint64_t flags; 1908 int r; 1909 1910 BUG_ON(adev->mman.buffer_funcs->copy_max_bytes < 1911 AMDGPU_GTT_MAX_TRANSFER_SIZE * 8); 1912 1913 *addr = adev->gmc.gart_start; 1914 *addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 1915 AMDGPU_GPU_PAGE_SIZE; 1916 1917 num_dw = adev->mman.buffer_funcs->copy_num_dw; 1918 while (num_dw & 0x7) 1919 num_dw++; 1920 1921 num_bytes = num_pages * 8; 1922 1923 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes, &job); 1924 if (r) 1925 return r; 1926 1927 src_addr = num_dw * 4; 1928 src_addr += job->ibs[0].gpu_addr; 1929 1930 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo); 1931 dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8; 1932 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, 1933 dst_addr, num_bytes); 1934 1935 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 1936 WARN_ON(job->ibs[0].length_dw > num_dw); 1937 1938 dma_address = >t->ttm.dma_address[offset >> PAGE_SHIFT]; 1939 flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem); 1940 r = amdgpu_gart_map(adev, 0, num_pages, dma_address, flags, 1941 &job->ibs[0].ptr[num_dw]); 1942 if (r) 1943 goto error_free; 1944 1945 r = amdgpu_job_submit(job, &adev->mman.entity, 1946 AMDGPU_FENCE_OWNER_UNDEFINED, &fence); 1947 if (r) 1948 goto error_free; 1949 1950 dma_fence_put(fence); 1951 1952 return r; 1953 1954 error_free: 1955 amdgpu_job_free(job); 1956 return r; 1957 } 1958 1959 int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset, 1960 uint64_t dst_offset, uint32_t byte_count, 1961 struct reservation_object *resv, 1962 struct dma_fence **fence, bool direct_submit, 1963 bool vm_needs_flush) 1964 { 1965 struct amdgpu_device *adev = ring->adev; 1966 struct amdgpu_job *job; 1967 1968 uint32_t max_bytes; 1969 unsigned num_loops, num_dw; 1970 unsigned i; 1971 int r; 1972 1973 if (direct_submit && !ring->sched.ready) { 1974 DRM_ERROR("Trying to move memory with ring turned off.\n"); 1975 return -EINVAL; 1976 } 1977 1978 max_bytes = adev->mman.buffer_funcs->copy_max_bytes; 1979 num_loops = DIV_ROUND_UP(byte_count, max_bytes); 1980 num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw; 1981 1982 /* for IB padding */ 1983 while (num_dw & 0x7) 1984 num_dw++; 1985 1986 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job); 1987 if (r) 1988 return r; 1989 1990 if (vm_needs_flush) { 1991 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo); 1992 job->vm_needs_flush = true; 1993 } 1994 if (resv) { 1995 r = amdgpu_sync_resv(adev, &job->sync, resv, 1996 AMDGPU_FENCE_OWNER_UNDEFINED, 1997 false); 1998 if (r) { 1999 DRM_ERROR("sync failed (%d).\n", r); 2000 goto error_free; 2001 } 2002 } 2003 2004 for (i = 0; i < num_loops; i++) { 2005 uint32_t cur_size_in_bytes = min(byte_count, max_bytes); 2006 2007 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset, 2008 dst_offset, cur_size_in_bytes); 2009 2010 src_offset += cur_size_in_bytes; 2011 dst_offset += cur_size_in_bytes; 2012 byte_count -= cur_size_in_bytes; 2013 } 2014 2015 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 2016 WARN_ON(job->ibs[0].length_dw > num_dw); 2017 if (direct_submit) 2018 r = amdgpu_job_submit_direct(job, ring, fence); 2019 else 2020 r = amdgpu_job_submit(job, &adev->mman.entity, 2021 AMDGPU_FENCE_OWNER_UNDEFINED, fence); 2022 if (r) 2023 goto error_free; 2024 2025 return r; 2026 2027 error_free: 2028 amdgpu_job_free(job); 2029 DRM_ERROR("Error scheduling IBs (%d)\n", r); 2030 return r; 2031 } 2032 2033 int amdgpu_fill_buffer(struct amdgpu_bo *bo, 2034 uint32_t src_data, 2035 struct reservation_object *resv, 2036 struct dma_fence **fence) 2037 { 2038 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 2039 uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes; 2040 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; 2041 2042 struct drm_mm_node *mm_node; 2043 unsigned long num_pages; 2044 unsigned int num_loops, num_dw; 2045 2046 struct amdgpu_job *job; 2047 int r; 2048 2049 if (!adev->mman.buffer_funcs_enabled) { 2050 DRM_ERROR("Trying to clear memory with ring turned off.\n"); 2051 return -EINVAL; 2052 } 2053 2054 if (bo->tbo.mem.mem_type == TTM_PL_TT) { 2055 r = amdgpu_ttm_alloc_gart(&bo->tbo); 2056 if (r) 2057 return r; 2058 } 2059 2060 num_pages = bo->tbo.num_pages; 2061 mm_node = bo->tbo.mem.mm_node; 2062 num_loops = 0; 2063 while (num_pages) { 2064 uint32_t byte_count = mm_node->size << PAGE_SHIFT; 2065 2066 num_loops += DIV_ROUND_UP(byte_count, max_bytes); 2067 num_pages -= mm_node->size; 2068 ++mm_node; 2069 } 2070 num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw; 2071 2072 /* for IB padding */ 2073 num_dw += 64; 2074 2075 r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job); 2076 if (r) 2077 return r; 2078 2079 if (resv) { 2080 r = amdgpu_sync_resv(adev, &job->sync, resv, 2081 AMDGPU_FENCE_OWNER_UNDEFINED, false); 2082 if (r) { 2083 DRM_ERROR("sync failed (%d).\n", r); 2084 goto error_free; 2085 } 2086 } 2087 2088 num_pages = bo->tbo.num_pages; 2089 mm_node = bo->tbo.mem.mm_node; 2090 2091 while (num_pages) { 2092 uint32_t byte_count = mm_node->size << PAGE_SHIFT; 2093 uint64_t dst_addr; 2094 2095 dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem); 2096 while (byte_count) { 2097 uint32_t cur_size_in_bytes = min(byte_count, max_bytes); 2098 2099 amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, 2100 dst_addr, cur_size_in_bytes); 2101 2102 dst_addr += cur_size_in_bytes; 2103 byte_count -= cur_size_in_bytes; 2104 } 2105 2106 num_pages -= mm_node->size; 2107 ++mm_node; 2108 } 2109 2110 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 2111 WARN_ON(job->ibs[0].length_dw > num_dw); 2112 r = amdgpu_job_submit(job, &adev->mman.entity, 2113 AMDGPU_FENCE_OWNER_UNDEFINED, fence); 2114 if (r) 2115 goto error_free; 2116 2117 return 0; 2118 2119 error_free: 2120 amdgpu_job_free(job); 2121 return r; 2122 } 2123 2124 #if defined(CONFIG_DEBUG_FS) 2125 2126 static int amdgpu_mm_dump_table(struct seq_file *m, void *data) 2127 { 2128 struct drm_info_node *node = (struct drm_info_node *)m->private; 2129 unsigned ttm_pl = (uintptr_t)node->info_ent->data; 2130 struct drm_device *dev = node->minor->dev; 2131 struct amdgpu_device *adev = dev->dev_private; 2132 struct ttm_mem_type_manager *man = &adev->mman.bdev.man[ttm_pl]; 2133 struct drm_printer p = drm_seq_file_printer(m); 2134 2135 man->func->debug(man, &p); 2136 return 0; 2137 } 2138 2139 static const struct drm_info_list amdgpu_ttm_debugfs_list[] = { 2140 {"amdgpu_vram_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_VRAM}, 2141 {"amdgpu_gtt_mm", amdgpu_mm_dump_table, 0, (void *)TTM_PL_TT}, 2142 {"amdgpu_gds_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GDS}, 2143 {"amdgpu_gws_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_GWS}, 2144 {"amdgpu_oa_mm", amdgpu_mm_dump_table, 0, (void *)AMDGPU_PL_OA}, 2145 {"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL}, 2146 #ifdef CONFIG_SWIOTLB 2147 {"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL} 2148 #endif 2149 }; 2150 2151 /** 2152 * amdgpu_ttm_vram_read - Linear read access to VRAM 2153 * 2154 * Accesses VRAM via MMIO for debugging purposes. 2155 */ 2156 static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf, 2157 size_t size, loff_t *pos) 2158 { 2159 struct amdgpu_device *adev = file_inode(f)->i_private; 2160 ssize_t result = 0; 2161 int r; 2162 2163 if (size & 0x3 || *pos & 0x3) 2164 return -EINVAL; 2165 2166 if (*pos >= adev->gmc.mc_vram_size) 2167 return -ENXIO; 2168 2169 while (size) { 2170 unsigned long flags; 2171 uint32_t value; 2172 2173 if (*pos >= adev->gmc.mc_vram_size) 2174 return result; 2175 2176 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 2177 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000); 2178 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31); 2179 value = RREG32_NO_KIQ(mmMM_DATA); 2180 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 2181 2182 r = put_user(value, (uint32_t *)buf); 2183 if (r) 2184 return r; 2185 2186 result += 4; 2187 buf += 4; 2188 *pos += 4; 2189 size -= 4; 2190 } 2191 2192 return result; 2193 } 2194 2195 /** 2196 * amdgpu_ttm_vram_write - Linear write access to VRAM 2197 * 2198 * Accesses VRAM via MMIO for debugging purposes. 2199 */ 2200 static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf, 2201 size_t size, loff_t *pos) 2202 { 2203 struct amdgpu_device *adev = file_inode(f)->i_private; 2204 ssize_t result = 0; 2205 int r; 2206 2207 if (size & 0x3 || *pos & 0x3) 2208 return -EINVAL; 2209 2210 if (*pos >= adev->gmc.mc_vram_size) 2211 return -ENXIO; 2212 2213 while (size) { 2214 unsigned long flags; 2215 uint32_t value; 2216 2217 if (*pos >= adev->gmc.mc_vram_size) 2218 return result; 2219 2220 r = get_user(value, (uint32_t *)buf); 2221 if (r) 2222 return r; 2223 2224 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 2225 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000); 2226 WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31); 2227 WREG32_NO_KIQ(mmMM_DATA, value); 2228 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 2229 2230 result += 4; 2231 buf += 4; 2232 *pos += 4; 2233 size -= 4; 2234 } 2235 2236 return result; 2237 } 2238 2239 static const struct file_operations amdgpu_ttm_vram_fops = { 2240 .owner = THIS_MODULE, 2241 .read = amdgpu_ttm_vram_read, 2242 .write = amdgpu_ttm_vram_write, 2243 .llseek = default_llseek, 2244 }; 2245 2246 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS 2247 2248 /** 2249 * amdgpu_ttm_gtt_read - Linear read access to GTT memory 2250 */ 2251 static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf, 2252 size_t size, loff_t *pos) 2253 { 2254 struct amdgpu_device *adev = file_inode(f)->i_private; 2255 ssize_t result = 0; 2256 int r; 2257 2258 while (size) { 2259 loff_t p = *pos / PAGE_SIZE; 2260 unsigned off = *pos & ~PAGE_MASK; 2261 size_t cur_size = min_t(size_t, size, PAGE_SIZE - off); 2262 struct page *page; 2263 void *ptr; 2264 2265 if (p >= adev->gart.num_cpu_pages) 2266 return result; 2267 2268 page = adev->gart.pages[p]; 2269 if (page) { 2270 ptr = kmap(page); 2271 ptr += off; 2272 2273 r = copy_to_user(buf, ptr, cur_size); 2274 kunmap(adev->gart.pages[p]); 2275 } else 2276 r = clear_user(buf, cur_size); 2277 2278 if (r) 2279 return -EFAULT; 2280 2281 result += cur_size; 2282 buf += cur_size; 2283 *pos += cur_size; 2284 size -= cur_size; 2285 } 2286 2287 return result; 2288 } 2289 2290 static const struct file_operations amdgpu_ttm_gtt_fops = { 2291 .owner = THIS_MODULE, 2292 .read = amdgpu_ttm_gtt_read, 2293 .llseek = default_llseek 2294 }; 2295 2296 #endif 2297 2298 /** 2299 * amdgpu_iomem_read - Virtual read access to GPU mapped memory 2300 * 2301 * This function is used to read memory that has been mapped to the 2302 * GPU and the known addresses are not physical addresses but instead 2303 * bus addresses (e.g., what you'd put in an IB or ring buffer). 2304 */ 2305 static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf, 2306 size_t size, loff_t *pos) 2307 { 2308 struct amdgpu_device *adev = file_inode(f)->i_private; 2309 struct iommu_domain *dom; 2310 ssize_t result = 0; 2311 int r; 2312 2313 /* retrieve the IOMMU domain if any for this device */ 2314 dom = iommu_get_domain_for_dev(adev->dev); 2315 2316 while (size) { 2317 phys_addr_t addr = *pos & PAGE_MASK; 2318 loff_t off = *pos & ~PAGE_MASK; 2319 size_t bytes = PAGE_SIZE - off; 2320 unsigned long pfn; 2321 struct page *p; 2322 void *ptr; 2323 2324 bytes = bytes < size ? bytes : size; 2325 2326 /* Translate the bus address to a physical address. If 2327 * the domain is NULL it means there is no IOMMU active 2328 * and the address translation is the identity 2329 */ 2330 addr = dom ? iommu_iova_to_phys(dom, addr) : addr; 2331 2332 pfn = addr >> PAGE_SHIFT; 2333 if (!pfn_valid(pfn)) 2334 return -EPERM; 2335 2336 p = pfn_to_page(pfn); 2337 if (p->mapping != adev->mman.bdev.dev_mapping) 2338 return -EPERM; 2339 2340 ptr = kmap(p); 2341 r = copy_to_user(buf, ptr + off, bytes); 2342 kunmap(p); 2343 if (r) 2344 return -EFAULT; 2345 2346 size -= bytes; 2347 *pos += bytes; 2348 result += bytes; 2349 } 2350 2351 return result; 2352 } 2353 2354 /** 2355 * amdgpu_iomem_write - Virtual write access to GPU mapped memory 2356 * 2357 * This function is used to write memory that has been mapped to the 2358 * GPU and the known addresses are not physical addresses but instead 2359 * bus addresses (e.g., what you'd put in an IB or ring buffer). 2360 */ 2361 static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf, 2362 size_t size, loff_t *pos) 2363 { 2364 struct amdgpu_device *adev = file_inode(f)->i_private; 2365 struct iommu_domain *dom; 2366 ssize_t result = 0; 2367 int r; 2368 2369 dom = iommu_get_domain_for_dev(adev->dev); 2370 2371 while (size) { 2372 phys_addr_t addr = *pos & PAGE_MASK; 2373 loff_t off = *pos & ~PAGE_MASK; 2374 size_t bytes = PAGE_SIZE - off; 2375 unsigned long pfn; 2376 struct page *p; 2377 void *ptr; 2378 2379 bytes = bytes < size ? bytes : size; 2380 2381 addr = dom ? iommu_iova_to_phys(dom, addr) : addr; 2382 2383 pfn = addr >> PAGE_SHIFT; 2384 if (!pfn_valid(pfn)) 2385 return -EPERM; 2386 2387 p = pfn_to_page(pfn); 2388 if (p->mapping != adev->mman.bdev.dev_mapping) 2389 return -EPERM; 2390 2391 ptr = kmap(p); 2392 r = copy_from_user(ptr + off, buf, bytes); 2393 kunmap(p); 2394 if (r) 2395 return -EFAULT; 2396 2397 size -= bytes; 2398 *pos += bytes; 2399 result += bytes; 2400 } 2401 2402 return result; 2403 } 2404 2405 static const struct file_operations amdgpu_ttm_iomem_fops = { 2406 .owner = THIS_MODULE, 2407 .read = amdgpu_iomem_read, 2408 .write = amdgpu_iomem_write, 2409 .llseek = default_llseek 2410 }; 2411 2412 static const struct { 2413 char *name; 2414 const struct file_operations *fops; 2415 int domain; 2416 } ttm_debugfs_entries[] = { 2417 { "amdgpu_vram", &amdgpu_ttm_vram_fops, TTM_PL_VRAM }, 2418 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS 2419 { "amdgpu_gtt", &amdgpu_ttm_gtt_fops, TTM_PL_TT }, 2420 #endif 2421 { "amdgpu_iomem", &amdgpu_ttm_iomem_fops, TTM_PL_SYSTEM }, 2422 }; 2423 2424 #endif 2425 2426 static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev) 2427 { 2428 #if defined(CONFIG_DEBUG_FS) 2429 unsigned count; 2430 2431 struct drm_minor *minor = adev->ddev->primary; 2432 struct dentry *ent, *root = minor->debugfs_root; 2433 2434 for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) { 2435 ent = debugfs_create_file( 2436 ttm_debugfs_entries[count].name, 2437 S_IFREG | S_IRUGO, root, 2438 adev, 2439 ttm_debugfs_entries[count].fops); 2440 if (IS_ERR(ent)) 2441 return PTR_ERR(ent); 2442 if (ttm_debugfs_entries[count].domain == TTM_PL_VRAM) 2443 i_size_write(ent->d_inode, adev->gmc.mc_vram_size); 2444 else if (ttm_debugfs_entries[count].domain == TTM_PL_TT) 2445 i_size_write(ent->d_inode, adev->gmc.gart_size); 2446 adev->mman.debugfs_entries[count] = ent; 2447 } 2448 2449 count = ARRAY_SIZE(amdgpu_ttm_debugfs_list); 2450 2451 #ifdef CONFIG_SWIOTLB 2452 if (!(adev->need_swiotlb && swiotlb_nr_tbl())) 2453 --count; 2454 #endif 2455 2456 return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count); 2457 #else 2458 return 0; 2459 #endif 2460 } 2461 2462 static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev) 2463 { 2464 #if defined(CONFIG_DEBUG_FS) 2465 unsigned i; 2466 2467 for (i = 0; i < ARRAY_SIZE(ttm_debugfs_entries); i++) 2468 debugfs_remove(adev->mman.debugfs_entries[i]); 2469 #endif 2470 } 2471