1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 33 #include <linux/dma-mapping.h> 34 #include <linux/iommu.h> 35 #include <linux/pagemap.h> 36 #include <linux/sched/task.h> 37 #include <linux/sched/mm.h> 38 #include <linux/seq_file.h> 39 #include <linux/slab.h> 40 #include <linux/swap.h> 41 #include <linux/swiotlb.h> 42 #include <linux/dma-buf.h> 43 #include <linux/sizes.h> 44 #include <linux/module.h> 45 46 #include <drm/drm_drv.h> 47 #include <drm/ttm/ttm_bo_api.h> 48 #include <drm/ttm/ttm_bo_driver.h> 49 #include <drm/ttm/ttm_placement.h> 50 #include <drm/ttm/ttm_range_manager.h> 51 52 #include <drm/amdgpu_drm.h> 53 #include <drm/drm_drv.h> 54 55 #include "amdgpu.h" 56 #include "amdgpu_object.h" 57 #include "amdgpu_trace.h" 58 #include "amdgpu_amdkfd.h" 59 #include "amdgpu_sdma.h" 60 #include "amdgpu_ras.h" 61 #include "amdgpu_hmm.h" 62 #include "amdgpu_atomfirmware.h" 63 #include "amdgpu_res_cursor.h" 64 #include "bif/bif_4_1_d.h" 65 66 MODULE_IMPORT_NS(DMA_BUF); 67 68 #define AMDGPU_TTM_VRAM_MAX_DW_READ (size_t)128 69 70 static int amdgpu_ttm_backend_bind(struct ttm_device *bdev, 71 struct ttm_tt *ttm, 72 struct ttm_resource *bo_mem); 73 static void amdgpu_ttm_backend_unbind(struct ttm_device *bdev, 74 struct ttm_tt *ttm); 75 76 static int amdgpu_ttm_init_on_chip(struct amdgpu_device *adev, 77 unsigned int type, 78 uint64_t size_in_page) 79 { 80 return ttm_range_man_init(&adev->mman.bdev, type, 81 false, size_in_page); 82 } 83 84 /** 85 * amdgpu_evict_flags - Compute placement flags 86 * 87 * @bo: The buffer object to evict 88 * @placement: Possible destination(s) for evicted BO 89 * 90 * Fill in placement data when ttm_bo_evict() is called 91 */ 92 static void amdgpu_evict_flags(struct ttm_buffer_object *bo, 93 struct ttm_placement *placement) 94 { 95 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 96 struct amdgpu_bo *abo; 97 static const struct ttm_place placements = { 98 .fpfn = 0, 99 .lpfn = 0, 100 .mem_type = TTM_PL_SYSTEM, 101 .flags = 0 102 }; 103 104 /* Don't handle scatter gather BOs */ 105 if (bo->type == ttm_bo_type_sg) { 106 placement->num_placement = 0; 107 placement->num_busy_placement = 0; 108 return; 109 } 110 111 /* Object isn't an AMDGPU object so ignore */ 112 if (!amdgpu_bo_is_amdgpu_bo(bo)) { 113 placement->placement = &placements; 114 placement->busy_placement = &placements; 115 placement->num_placement = 1; 116 placement->num_busy_placement = 1; 117 return; 118 } 119 120 abo = ttm_to_amdgpu_bo(bo); 121 if (abo->flags & AMDGPU_GEM_CREATE_DISCARDABLE) { 122 placement->num_placement = 0; 123 placement->num_busy_placement = 0; 124 return; 125 } 126 127 switch (bo->resource->mem_type) { 128 case AMDGPU_PL_GDS: 129 case AMDGPU_PL_GWS: 130 case AMDGPU_PL_OA: 131 placement->num_placement = 0; 132 placement->num_busy_placement = 0; 133 return; 134 135 case TTM_PL_VRAM: 136 if (!adev->mman.buffer_funcs_enabled) { 137 /* Move to system memory */ 138 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); 139 } else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 140 !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) && 141 amdgpu_bo_in_cpu_visible_vram(abo)) { 142 143 /* Try evicting to the CPU inaccessible part of VRAM 144 * first, but only set GTT as busy placement, so this 145 * BO will be evicted to GTT rather than causing other 146 * BOs to be evicted from VRAM 147 */ 148 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | 149 AMDGPU_GEM_DOMAIN_GTT | 150 AMDGPU_GEM_DOMAIN_CPU); 151 abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; 152 abo->placements[0].lpfn = 0; 153 abo->placement.busy_placement = &abo->placements[1]; 154 abo->placement.num_busy_placement = 1; 155 } else { 156 /* Move to GTT memory */ 157 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT | 158 AMDGPU_GEM_DOMAIN_CPU); 159 } 160 break; 161 case TTM_PL_TT: 162 case AMDGPU_PL_PREEMPT: 163 default: 164 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU); 165 break; 166 } 167 *placement = abo->placement; 168 } 169 170 /** 171 * amdgpu_ttm_map_buffer - Map memory into the GART windows 172 * @bo: buffer object to map 173 * @mem: memory object to map 174 * @mm_cur: range to map 175 * @window: which GART window to use 176 * @ring: DMA ring to use for the copy 177 * @tmz: if we should setup a TMZ enabled mapping 178 * @size: in number of bytes to map, out number of bytes mapped 179 * @addr: resulting address inside the MC address space 180 * 181 * Setup one of the GART windows to access a specific piece of memory or return 182 * the physical address for local memory. 183 */ 184 static int amdgpu_ttm_map_buffer(struct ttm_buffer_object *bo, 185 struct ttm_resource *mem, 186 struct amdgpu_res_cursor *mm_cur, 187 unsigned window, struct amdgpu_ring *ring, 188 bool tmz, uint64_t *size, uint64_t *addr) 189 { 190 struct amdgpu_device *adev = ring->adev; 191 unsigned offset, num_pages, num_dw, num_bytes; 192 uint64_t src_addr, dst_addr; 193 struct amdgpu_job *job; 194 void *cpu_addr; 195 uint64_t flags; 196 unsigned int i; 197 int r; 198 199 BUG_ON(adev->mman.buffer_funcs->copy_max_bytes < 200 AMDGPU_GTT_MAX_TRANSFER_SIZE * 8); 201 202 if (WARN_ON(mem->mem_type == AMDGPU_PL_PREEMPT)) 203 return -EINVAL; 204 205 /* Map only what can't be accessed directly */ 206 if (!tmz && mem->start != AMDGPU_BO_INVALID_OFFSET) { 207 *addr = amdgpu_ttm_domain_start(adev, mem->mem_type) + 208 mm_cur->start; 209 return 0; 210 } 211 212 213 /* 214 * If start begins at an offset inside the page, then adjust the size 215 * and addr accordingly 216 */ 217 offset = mm_cur->start & ~PAGE_MASK; 218 219 num_pages = PFN_UP(*size + offset); 220 num_pages = min_t(uint32_t, num_pages, AMDGPU_GTT_MAX_TRANSFER_SIZE); 221 222 *size = min(*size, (uint64_t)num_pages * PAGE_SIZE - offset); 223 224 *addr = adev->gmc.gart_start; 225 *addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 226 AMDGPU_GPU_PAGE_SIZE; 227 *addr += offset; 228 229 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); 230 num_bytes = num_pages * 8 * AMDGPU_GPU_PAGES_IN_CPU_PAGE; 231 232 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.entity, 233 AMDGPU_FENCE_OWNER_UNDEFINED, 234 num_dw * 4 + num_bytes, 235 AMDGPU_IB_POOL_DELAYED, &job); 236 if (r) 237 return r; 238 239 src_addr = num_dw * 4; 240 src_addr += job->ibs[0].gpu_addr; 241 242 dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo); 243 dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8; 244 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, 245 dst_addr, num_bytes, false); 246 247 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 248 WARN_ON(job->ibs[0].length_dw > num_dw); 249 250 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, mem); 251 if (tmz) 252 flags |= AMDGPU_PTE_TMZ; 253 254 cpu_addr = &job->ibs[0].ptr[num_dw]; 255 256 if (mem->mem_type == TTM_PL_TT) { 257 dma_addr_t *dma_addr; 258 259 dma_addr = &bo->ttm->dma_address[mm_cur->start >> PAGE_SHIFT]; 260 amdgpu_gart_map(adev, 0, num_pages, dma_addr, flags, cpu_addr); 261 } else { 262 dma_addr_t dma_address; 263 264 dma_address = mm_cur->start; 265 dma_address += adev->vm_manager.vram_base_offset; 266 267 for (i = 0; i < num_pages; ++i) { 268 amdgpu_gart_map(adev, i << PAGE_SHIFT, 1, &dma_address, 269 flags, cpu_addr); 270 dma_address += PAGE_SIZE; 271 } 272 } 273 274 dma_fence_put(amdgpu_job_submit(job)); 275 return 0; 276 } 277 278 /** 279 * amdgpu_ttm_copy_mem_to_mem - Helper function for copy 280 * @adev: amdgpu device 281 * @src: buffer/address where to read from 282 * @dst: buffer/address where to write to 283 * @size: number of bytes to copy 284 * @tmz: if a secure copy should be used 285 * @resv: resv object to sync to 286 * @f: Returns the last fence if multiple jobs are submitted. 287 * 288 * The function copies @size bytes from {src->mem + src->offset} to 289 * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a 290 * move and different for a BO to BO copy. 291 * 292 */ 293 int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev, 294 const struct amdgpu_copy_mem *src, 295 const struct amdgpu_copy_mem *dst, 296 uint64_t size, bool tmz, 297 struct dma_resv *resv, 298 struct dma_fence **f) 299 { 300 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; 301 struct amdgpu_res_cursor src_mm, dst_mm; 302 struct dma_fence *fence = NULL; 303 int r = 0; 304 305 if (!adev->mman.buffer_funcs_enabled) { 306 DRM_ERROR("Trying to move memory with ring turned off.\n"); 307 return -EINVAL; 308 } 309 310 amdgpu_res_first(src->mem, src->offset, size, &src_mm); 311 amdgpu_res_first(dst->mem, dst->offset, size, &dst_mm); 312 313 mutex_lock(&adev->mman.gtt_window_lock); 314 while (src_mm.remaining) { 315 uint64_t from, to, cur_size; 316 struct dma_fence *next; 317 318 /* Never copy more than 256MiB at once to avoid a timeout */ 319 cur_size = min3(src_mm.size, dst_mm.size, 256ULL << 20); 320 321 /* Map src to window 0 and dst to window 1. */ 322 r = amdgpu_ttm_map_buffer(src->bo, src->mem, &src_mm, 323 0, ring, tmz, &cur_size, &from); 324 if (r) 325 goto error; 326 327 r = amdgpu_ttm_map_buffer(dst->bo, dst->mem, &dst_mm, 328 1, ring, tmz, &cur_size, &to); 329 if (r) 330 goto error; 331 332 r = amdgpu_copy_buffer(ring, from, to, cur_size, 333 resv, &next, false, true, tmz); 334 if (r) 335 goto error; 336 337 dma_fence_put(fence); 338 fence = next; 339 340 amdgpu_res_next(&src_mm, cur_size); 341 amdgpu_res_next(&dst_mm, cur_size); 342 } 343 error: 344 mutex_unlock(&adev->mman.gtt_window_lock); 345 if (f) 346 *f = dma_fence_get(fence); 347 dma_fence_put(fence); 348 return r; 349 } 350 351 /* 352 * amdgpu_move_blit - Copy an entire buffer to another buffer 353 * 354 * This is a helper called by amdgpu_bo_move() and amdgpu_move_vram_ram() to 355 * help move buffers to and from VRAM. 356 */ 357 static int amdgpu_move_blit(struct ttm_buffer_object *bo, 358 bool evict, 359 struct ttm_resource *new_mem, 360 struct ttm_resource *old_mem) 361 { 362 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 363 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 364 struct amdgpu_copy_mem src, dst; 365 struct dma_fence *fence = NULL; 366 int r; 367 368 src.bo = bo; 369 dst.bo = bo; 370 src.mem = old_mem; 371 dst.mem = new_mem; 372 src.offset = 0; 373 dst.offset = 0; 374 375 r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst, 376 new_mem->size, 377 amdgpu_bo_encrypted(abo), 378 bo->base.resv, &fence); 379 if (r) 380 goto error; 381 382 /* clear the space being freed */ 383 if (old_mem->mem_type == TTM_PL_VRAM && 384 (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) { 385 struct dma_fence *wipe_fence = NULL; 386 387 r = amdgpu_fill_buffer(abo, AMDGPU_POISON, NULL, &wipe_fence); 388 if (r) { 389 goto error; 390 } else if (wipe_fence) { 391 dma_fence_put(fence); 392 fence = wipe_fence; 393 } 394 } 395 396 /* Always block for VM page tables before committing the new location */ 397 if (bo->type == ttm_bo_type_kernel) 398 r = ttm_bo_move_accel_cleanup(bo, fence, true, false, new_mem); 399 else 400 r = ttm_bo_move_accel_cleanup(bo, fence, evict, true, new_mem); 401 dma_fence_put(fence); 402 return r; 403 404 error: 405 if (fence) 406 dma_fence_wait(fence, false); 407 dma_fence_put(fence); 408 return r; 409 } 410 411 /* 412 * amdgpu_mem_visible - Check that memory can be accessed by ttm_bo_move_memcpy 413 * 414 * Called by amdgpu_bo_move() 415 */ 416 static bool amdgpu_mem_visible(struct amdgpu_device *adev, 417 struct ttm_resource *mem) 418 { 419 u64 mem_size = (u64)mem->size; 420 struct amdgpu_res_cursor cursor; 421 u64 end; 422 423 if (mem->mem_type == TTM_PL_SYSTEM || 424 mem->mem_type == TTM_PL_TT) 425 return true; 426 if (mem->mem_type != TTM_PL_VRAM) 427 return false; 428 429 amdgpu_res_first(mem, 0, mem_size, &cursor); 430 end = cursor.start + cursor.size; 431 while (cursor.remaining) { 432 amdgpu_res_next(&cursor, cursor.size); 433 434 if (!cursor.remaining) 435 break; 436 437 /* ttm_resource_ioremap only supports contiguous memory */ 438 if (end != cursor.start) 439 return false; 440 441 end = cursor.start + cursor.size; 442 } 443 444 return end <= adev->gmc.visible_vram_size; 445 } 446 447 /* 448 * amdgpu_bo_move - Move a buffer object to a new memory location 449 * 450 * Called by ttm_bo_handle_move_mem() 451 */ 452 static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, 453 struct ttm_operation_ctx *ctx, 454 struct ttm_resource *new_mem, 455 struct ttm_place *hop) 456 { 457 struct amdgpu_device *adev; 458 struct amdgpu_bo *abo; 459 struct ttm_resource *old_mem = bo->resource; 460 int r; 461 462 if (new_mem->mem_type == TTM_PL_TT || 463 new_mem->mem_type == AMDGPU_PL_PREEMPT) { 464 r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem); 465 if (r) 466 return r; 467 } 468 469 /* Can't move a pinned BO */ 470 abo = ttm_to_amdgpu_bo(bo); 471 if (WARN_ON_ONCE(abo->tbo.pin_count > 0)) 472 return -EINVAL; 473 474 adev = amdgpu_ttm_adev(bo->bdev); 475 476 if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM && 477 bo->ttm == NULL)) { 478 ttm_bo_move_null(bo, new_mem); 479 goto out; 480 } 481 if (old_mem->mem_type == TTM_PL_SYSTEM && 482 (new_mem->mem_type == TTM_PL_TT || 483 new_mem->mem_type == AMDGPU_PL_PREEMPT)) { 484 ttm_bo_move_null(bo, new_mem); 485 goto out; 486 } 487 if ((old_mem->mem_type == TTM_PL_TT || 488 old_mem->mem_type == AMDGPU_PL_PREEMPT) && 489 new_mem->mem_type == TTM_PL_SYSTEM) { 490 r = ttm_bo_wait_ctx(bo, ctx); 491 if (r) 492 return r; 493 494 amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm); 495 ttm_resource_free(bo, &bo->resource); 496 ttm_bo_assign_mem(bo, new_mem); 497 goto out; 498 } 499 500 if (old_mem->mem_type == AMDGPU_PL_GDS || 501 old_mem->mem_type == AMDGPU_PL_GWS || 502 old_mem->mem_type == AMDGPU_PL_OA || 503 new_mem->mem_type == AMDGPU_PL_GDS || 504 new_mem->mem_type == AMDGPU_PL_GWS || 505 new_mem->mem_type == AMDGPU_PL_OA) { 506 /* Nothing to save here */ 507 ttm_bo_move_null(bo, new_mem); 508 goto out; 509 } 510 511 if (bo->type == ttm_bo_type_device && 512 new_mem->mem_type == TTM_PL_VRAM && 513 old_mem->mem_type != TTM_PL_VRAM) { 514 /* amdgpu_bo_fault_reserve_notify will re-set this if the CPU 515 * accesses the BO after it's moved. 516 */ 517 abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 518 } 519 520 if (adev->mman.buffer_funcs_enabled) { 521 if (((old_mem->mem_type == TTM_PL_SYSTEM && 522 new_mem->mem_type == TTM_PL_VRAM) || 523 (old_mem->mem_type == TTM_PL_VRAM && 524 new_mem->mem_type == TTM_PL_SYSTEM))) { 525 hop->fpfn = 0; 526 hop->lpfn = 0; 527 hop->mem_type = TTM_PL_TT; 528 hop->flags = TTM_PL_FLAG_TEMPORARY; 529 return -EMULTIHOP; 530 } 531 532 r = amdgpu_move_blit(bo, evict, new_mem, old_mem); 533 } else { 534 r = -ENODEV; 535 } 536 537 if (r) { 538 /* Check that all memory is CPU accessible */ 539 if (!amdgpu_mem_visible(adev, old_mem) || 540 !amdgpu_mem_visible(adev, new_mem)) { 541 pr_err("Move buffer fallback to memcpy unavailable\n"); 542 return r; 543 } 544 545 r = ttm_bo_move_memcpy(bo, ctx, new_mem); 546 if (r) 547 return r; 548 } 549 550 out: 551 /* update statistics */ 552 atomic64_add(bo->base.size, &adev->num_bytes_moved); 553 amdgpu_bo_move_notify(bo, evict, new_mem); 554 return 0; 555 } 556 557 /* 558 * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault 559 * 560 * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault() 561 */ 562 static int amdgpu_ttm_io_mem_reserve(struct ttm_device *bdev, 563 struct ttm_resource *mem) 564 { 565 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 566 size_t bus_size = (size_t)mem->size; 567 568 switch (mem->mem_type) { 569 case TTM_PL_SYSTEM: 570 /* system memory */ 571 return 0; 572 case TTM_PL_TT: 573 case AMDGPU_PL_PREEMPT: 574 break; 575 case TTM_PL_VRAM: 576 mem->bus.offset = mem->start << PAGE_SHIFT; 577 /* check if it's visible */ 578 if ((mem->bus.offset + bus_size) > adev->gmc.visible_vram_size) 579 return -EINVAL; 580 581 if (adev->mman.aper_base_kaddr && 582 mem->placement & TTM_PL_FLAG_CONTIGUOUS) 583 mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr + 584 mem->bus.offset; 585 586 mem->bus.offset += adev->gmc.aper_base; 587 mem->bus.is_iomem = true; 588 break; 589 default: 590 return -EINVAL; 591 } 592 return 0; 593 } 594 595 static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo, 596 unsigned long page_offset) 597 { 598 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 599 struct amdgpu_res_cursor cursor; 600 601 amdgpu_res_first(bo->resource, (u64)page_offset << PAGE_SHIFT, 0, 602 &cursor); 603 return (adev->gmc.aper_base + cursor.start) >> PAGE_SHIFT; 604 } 605 606 /** 607 * amdgpu_ttm_domain_start - Returns GPU start address 608 * @adev: amdgpu device object 609 * @type: type of the memory 610 * 611 * Returns: 612 * GPU start address of a memory domain 613 */ 614 615 uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type) 616 { 617 switch (type) { 618 case TTM_PL_TT: 619 return adev->gmc.gart_start; 620 case TTM_PL_VRAM: 621 return adev->gmc.vram_start; 622 } 623 624 return 0; 625 } 626 627 /* 628 * TTM backend functions. 629 */ 630 struct amdgpu_ttm_tt { 631 struct ttm_tt ttm; 632 struct drm_gem_object *gobj; 633 u64 offset; 634 uint64_t userptr; 635 struct task_struct *usertask; 636 uint32_t userflags; 637 bool bound; 638 }; 639 640 #define ttm_to_amdgpu_ttm_tt(ptr) container_of(ptr, struct amdgpu_ttm_tt, ttm) 641 642 #ifdef CONFIG_DRM_AMDGPU_USERPTR 643 /* 644 * amdgpu_ttm_tt_get_user_pages - get device accessible pages that back user 645 * memory and start HMM tracking CPU page table update 646 * 647 * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only 648 * once afterwards to stop HMM tracking 649 */ 650 int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages, 651 struct hmm_range **range) 652 { 653 struct ttm_tt *ttm = bo->tbo.ttm; 654 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 655 unsigned long start = gtt->userptr; 656 struct vm_area_struct *vma; 657 struct mm_struct *mm; 658 bool readonly; 659 int r = 0; 660 661 /* Make sure get_user_pages_done() can cleanup gracefully */ 662 *range = NULL; 663 664 mm = bo->notifier.mm; 665 if (unlikely(!mm)) { 666 DRM_DEBUG_DRIVER("BO is not registered?\n"); 667 return -EFAULT; 668 } 669 670 if (!mmget_not_zero(mm)) /* Happens during process shutdown */ 671 return -ESRCH; 672 673 mmap_read_lock(mm); 674 vma = vma_lookup(mm, start); 675 if (unlikely(!vma)) { 676 r = -EFAULT; 677 goto out_unlock; 678 } 679 if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) && 680 vma->vm_file)) { 681 r = -EPERM; 682 goto out_unlock; 683 } 684 685 readonly = amdgpu_ttm_tt_is_readonly(ttm); 686 r = amdgpu_hmm_range_get_pages(&bo->notifier, start, ttm->num_pages, 687 readonly, NULL, pages, range); 688 out_unlock: 689 mmap_read_unlock(mm); 690 if (r) 691 pr_debug("failed %d to get user pages 0x%lx\n", r, start); 692 693 mmput(mm); 694 695 return r; 696 } 697 698 /* 699 * amdgpu_ttm_tt_userptr_range_done - stop HMM track the CPU page table change 700 * Check if the pages backing this ttm range have been invalidated 701 * 702 * Returns: true if pages are still valid 703 */ 704 bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm, 705 struct hmm_range *range) 706 { 707 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 708 709 if (!gtt || !gtt->userptr || !range) 710 return false; 711 712 DRM_DEBUG_DRIVER("user_pages_done 0x%llx pages 0x%x\n", 713 gtt->userptr, ttm->num_pages); 714 715 WARN_ONCE(!range->hmm_pfns, "No user pages to check\n"); 716 717 /* 718 * FIXME: Must always hold notifier_lock for this, and must 719 * not ignore the return code. 720 */ 721 return !amdgpu_hmm_range_get_pages_done(range); 722 } 723 #endif 724 725 /* 726 * amdgpu_ttm_tt_set_user_pages - Copy pages in, putting old pages as necessary. 727 * 728 * Called by amdgpu_cs_list_validate(). This creates the page list 729 * that backs user memory and will ultimately be mapped into the device 730 * address space. 731 */ 732 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages) 733 { 734 unsigned long i; 735 736 for (i = 0; i < ttm->num_pages; ++i) 737 ttm->pages[i] = pages ? pages[i] : NULL; 738 } 739 740 /* 741 * amdgpu_ttm_tt_pin_userptr - prepare the sg table with the user pages 742 * 743 * Called by amdgpu_ttm_backend_bind() 744 **/ 745 static int amdgpu_ttm_tt_pin_userptr(struct ttm_device *bdev, 746 struct ttm_tt *ttm) 747 { 748 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 749 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 750 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); 751 enum dma_data_direction direction = write ? 752 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 753 int r; 754 755 /* Allocate an SG array and squash pages into it */ 756 r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0, 757 (u64)ttm->num_pages << PAGE_SHIFT, 758 GFP_KERNEL); 759 if (r) 760 goto release_sg; 761 762 /* Map SG to device */ 763 r = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); 764 if (r) 765 goto release_sg; 766 767 /* convert SG to linear array of pages and dma addresses */ 768 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, 769 ttm->num_pages); 770 771 return 0; 772 773 release_sg: 774 kfree(ttm->sg); 775 ttm->sg = NULL; 776 return r; 777 } 778 779 /* 780 * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages 781 */ 782 static void amdgpu_ttm_tt_unpin_userptr(struct ttm_device *bdev, 783 struct ttm_tt *ttm) 784 { 785 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 786 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 787 int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); 788 enum dma_data_direction direction = write ? 789 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 790 791 /* double check that we don't free the table twice */ 792 if (!ttm->sg || !ttm->sg->sgl) 793 return; 794 795 /* unmap the pages mapped to the device */ 796 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); 797 sg_free_table(ttm->sg); 798 } 799 800 static void amdgpu_ttm_gart_bind(struct amdgpu_device *adev, 801 struct ttm_buffer_object *tbo, 802 uint64_t flags) 803 { 804 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo); 805 struct ttm_tt *ttm = tbo->ttm; 806 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 807 808 if (amdgpu_bo_encrypted(abo)) 809 flags |= AMDGPU_PTE_TMZ; 810 811 if (abo->flags & AMDGPU_GEM_CREATE_CP_MQD_GFX9) { 812 uint64_t page_idx = 1; 813 814 amdgpu_gart_bind(adev, gtt->offset, page_idx, 815 gtt->ttm.dma_address, flags); 816 817 /* The memory type of the first page defaults to UC. Now 818 * modify the memory type to NC from the second page of 819 * the BO onward. 820 */ 821 flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; 822 flags |= AMDGPU_PTE_MTYPE_VG10(AMDGPU_MTYPE_NC); 823 824 amdgpu_gart_bind(adev, gtt->offset + (page_idx << PAGE_SHIFT), 825 ttm->num_pages - page_idx, 826 &(gtt->ttm.dma_address[page_idx]), flags); 827 } else { 828 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, 829 gtt->ttm.dma_address, flags); 830 } 831 } 832 833 /* 834 * amdgpu_ttm_backend_bind - Bind GTT memory 835 * 836 * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem(). 837 * This handles binding GTT memory to the device address space. 838 */ 839 static int amdgpu_ttm_backend_bind(struct ttm_device *bdev, 840 struct ttm_tt *ttm, 841 struct ttm_resource *bo_mem) 842 { 843 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 844 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 845 uint64_t flags; 846 int r; 847 848 if (!bo_mem) 849 return -EINVAL; 850 851 if (gtt->bound) 852 return 0; 853 854 if (gtt->userptr) { 855 r = amdgpu_ttm_tt_pin_userptr(bdev, ttm); 856 if (r) { 857 DRM_ERROR("failed to pin userptr\n"); 858 return r; 859 } 860 } else if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) { 861 if (!ttm->sg) { 862 struct dma_buf_attachment *attach; 863 struct sg_table *sgt; 864 865 attach = gtt->gobj->import_attach; 866 sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); 867 if (IS_ERR(sgt)) 868 return PTR_ERR(sgt); 869 870 ttm->sg = sgt; 871 } 872 873 drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, 874 ttm->num_pages); 875 } 876 877 if (!ttm->num_pages) { 878 WARN(1, "nothing to bind %u pages for mreg %p back %p!\n", 879 ttm->num_pages, bo_mem, ttm); 880 } 881 882 if (bo_mem->mem_type != TTM_PL_TT || 883 !amdgpu_gtt_mgr_has_gart_addr(bo_mem)) { 884 gtt->offset = AMDGPU_BO_INVALID_OFFSET; 885 return 0; 886 } 887 888 /* compute PTE flags relevant to this BO memory */ 889 flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem); 890 891 /* bind pages into GART page tables */ 892 gtt->offset = (u64)bo_mem->start << PAGE_SHIFT; 893 amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages, 894 gtt->ttm.dma_address, flags); 895 gtt->bound = true; 896 return 0; 897 } 898 899 /* 900 * amdgpu_ttm_alloc_gart - Make sure buffer object is accessible either 901 * through AGP or GART aperture. 902 * 903 * If bo is accessible through AGP aperture, then use AGP aperture 904 * to access bo; otherwise allocate logical space in GART aperture 905 * and map bo to GART aperture. 906 */ 907 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo) 908 { 909 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 910 struct ttm_operation_ctx ctx = { false, false }; 911 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(bo->ttm); 912 struct ttm_placement placement; 913 struct ttm_place placements; 914 struct ttm_resource *tmp; 915 uint64_t addr, flags; 916 int r; 917 918 if (bo->resource->start != AMDGPU_BO_INVALID_OFFSET) 919 return 0; 920 921 addr = amdgpu_gmc_agp_addr(bo); 922 if (addr != AMDGPU_BO_INVALID_OFFSET) { 923 bo->resource->start = addr >> PAGE_SHIFT; 924 return 0; 925 } 926 927 /* allocate GART space */ 928 placement.num_placement = 1; 929 placement.placement = &placements; 930 placement.num_busy_placement = 1; 931 placement.busy_placement = &placements; 932 placements.fpfn = 0; 933 placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT; 934 placements.mem_type = TTM_PL_TT; 935 placements.flags = bo->resource->placement; 936 937 r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx); 938 if (unlikely(r)) 939 return r; 940 941 /* compute PTE flags for this buffer object */ 942 flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, tmp); 943 944 /* Bind pages */ 945 gtt->offset = (u64)tmp->start << PAGE_SHIFT; 946 amdgpu_ttm_gart_bind(adev, bo, flags); 947 amdgpu_gart_invalidate_tlb(adev); 948 ttm_resource_free(bo, &bo->resource); 949 ttm_bo_assign_mem(bo, tmp); 950 951 return 0; 952 } 953 954 /* 955 * amdgpu_ttm_recover_gart - Rebind GTT pages 956 * 957 * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to 958 * rebind GTT pages during a GPU reset. 959 */ 960 void amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo) 961 { 962 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); 963 uint64_t flags; 964 965 if (!tbo->ttm) 966 return; 967 968 flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, tbo->resource); 969 amdgpu_ttm_gart_bind(adev, tbo, flags); 970 } 971 972 /* 973 * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages 974 * 975 * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and 976 * ttm_tt_destroy(). 977 */ 978 static void amdgpu_ttm_backend_unbind(struct ttm_device *bdev, 979 struct ttm_tt *ttm) 980 { 981 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 982 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 983 984 /* if the pages have userptr pinning then clear that first */ 985 if (gtt->userptr) { 986 amdgpu_ttm_tt_unpin_userptr(bdev, ttm); 987 } else if (ttm->sg && gtt->gobj->import_attach) { 988 struct dma_buf_attachment *attach; 989 990 attach = gtt->gobj->import_attach; 991 dma_buf_unmap_attachment(attach, ttm->sg, DMA_BIDIRECTIONAL); 992 ttm->sg = NULL; 993 } 994 995 if (!gtt->bound) 996 return; 997 998 if (gtt->offset == AMDGPU_BO_INVALID_OFFSET) 999 return; 1000 1001 /* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */ 1002 amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages); 1003 gtt->bound = false; 1004 } 1005 1006 static void amdgpu_ttm_backend_destroy(struct ttm_device *bdev, 1007 struct ttm_tt *ttm) 1008 { 1009 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 1010 1011 if (gtt->usertask) 1012 put_task_struct(gtt->usertask); 1013 1014 ttm_tt_fini(>t->ttm); 1015 kfree(gtt); 1016 } 1017 1018 /** 1019 * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO 1020 * 1021 * @bo: The buffer object to create a GTT ttm_tt object around 1022 * @page_flags: Page flags to be added to the ttm_tt object 1023 * 1024 * Called by ttm_tt_create(). 1025 */ 1026 static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo, 1027 uint32_t page_flags) 1028 { 1029 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 1030 struct amdgpu_ttm_tt *gtt; 1031 enum ttm_caching caching; 1032 1033 gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL); 1034 if (gtt == NULL) { 1035 return NULL; 1036 } 1037 gtt->gobj = &bo->base; 1038 1039 if (abo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 1040 caching = ttm_write_combined; 1041 else 1042 caching = ttm_cached; 1043 1044 /* allocate space for the uninitialized page entries */ 1045 if (ttm_sg_tt_init(>t->ttm, bo, page_flags, caching)) { 1046 kfree(gtt); 1047 return NULL; 1048 } 1049 return >t->ttm; 1050 } 1051 1052 /* 1053 * amdgpu_ttm_tt_populate - Map GTT pages visible to the device 1054 * 1055 * Map the pages of a ttm_tt object to an address space visible 1056 * to the underlying device. 1057 */ 1058 static int amdgpu_ttm_tt_populate(struct ttm_device *bdev, 1059 struct ttm_tt *ttm, 1060 struct ttm_operation_ctx *ctx) 1061 { 1062 struct amdgpu_device *adev = amdgpu_ttm_adev(bdev); 1063 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 1064 pgoff_t i; 1065 int ret; 1066 1067 /* user pages are bound by amdgpu_ttm_tt_pin_userptr() */ 1068 if (gtt->userptr) { 1069 ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL); 1070 if (!ttm->sg) 1071 return -ENOMEM; 1072 return 0; 1073 } 1074 1075 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) 1076 return 0; 1077 1078 ret = ttm_pool_alloc(&adev->mman.bdev.pool, ttm, ctx); 1079 if (ret) 1080 return ret; 1081 1082 for (i = 0; i < ttm->num_pages; ++i) 1083 ttm->pages[i]->mapping = bdev->dev_mapping; 1084 1085 return 0; 1086 } 1087 1088 /* 1089 * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays 1090 * 1091 * Unmaps pages of a ttm_tt object from the device address space and 1092 * unpopulates the page array backing it. 1093 */ 1094 static void amdgpu_ttm_tt_unpopulate(struct ttm_device *bdev, 1095 struct ttm_tt *ttm) 1096 { 1097 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 1098 struct amdgpu_device *adev; 1099 pgoff_t i; 1100 1101 amdgpu_ttm_backend_unbind(bdev, ttm); 1102 1103 if (gtt->userptr) { 1104 amdgpu_ttm_tt_set_user_pages(ttm, NULL); 1105 kfree(ttm->sg); 1106 ttm->sg = NULL; 1107 return; 1108 } 1109 1110 if (ttm->page_flags & TTM_TT_FLAG_EXTERNAL) 1111 return; 1112 1113 for (i = 0; i < ttm->num_pages; ++i) 1114 ttm->pages[i]->mapping = NULL; 1115 1116 adev = amdgpu_ttm_adev(bdev); 1117 return ttm_pool_free(&adev->mman.bdev.pool, ttm); 1118 } 1119 1120 /** 1121 * amdgpu_ttm_tt_get_userptr - Return the userptr GTT ttm_tt for the current 1122 * task 1123 * 1124 * @tbo: The ttm_buffer_object that contains the userptr 1125 * @user_addr: The returned value 1126 */ 1127 int amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object *tbo, 1128 uint64_t *user_addr) 1129 { 1130 struct amdgpu_ttm_tt *gtt; 1131 1132 if (!tbo->ttm) 1133 return -EINVAL; 1134 1135 gtt = (void *)tbo->ttm; 1136 *user_addr = gtt->userptr; 1137 return 0; 1138 } 1139 1140 /** 1141 * amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current 1142 * task 1143 * 1144 * @bo: The ttm_buffer_object to bind this userptr to 1145 * @addr: The address in the current tasks VM space to use 1146 * @flags: Requirements of userptr object. 1147 * 1148 * Called by amdgpu_gem_userptr_ioctl() and kfd_ioctl_alloc_memory_of_gpu() to 1149 * bind userptr pages to current task and by kfd_ioctl_acquire_vm() to 1150 * initialize GPU VM for a KFD process. 1151 */ 1152 int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo, 1153 uint64_t addr, uint32_t flags) 1154 { 1155 struct amdgpu_ttm_tt *gtt; 1156 1157 if (!bo->ttm) { 1158 /* TODO: We want a separate TTM object type for userptrs */ 1159 bo->ttm = amdgpu_ttm_tt_create(bo, 0); 1160 if (bo->ttm == NULL) 1161 return -ENOMEM; 1162 } 1163 1164 /* Set TTM_TT_FLAG_EXTERNAL before populate but after create. */ 1165 bo->ttm->page_flags |= TTM_TT_FLAG_EXTERNAL; 1166 1167 gtt = ttm_to_amdgpu_ttm_tt(bo->ttm); 1168 gtt->userptr = addr; 1169 gtt->userflags = flags; 1170 1171 if (gtt->usertask) 1172 put_task_struct(gtt->usertask); 1173 gtt->usertask = current->group_leader; 1174 get_task_struct(gtt->usertask); 1175 1176 return 0; 1177 } 1178 1179 /* 1180 * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object 1181 */ 1182 struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm) 1183 { 1184 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 1185 1186 if (gtt == NULL) 1187 return NULL; 1188 1189 if (gtt->usertask == NULL) 1190 return NULL; 1191 1192 return gtt->usertask->mm; 1193 } 1194 1195 /* 1196 * amdgpu_ttm_tt_affect_userptr - Determine if a ttm_tt object lays inside an 1197 * address range for the current task. 1198 * 1199 */ 1200 bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, 1201 unsigned long end, unsigned long *userptr) 1202 { 1203 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 1204 unsigned long size; 1205 1206 if (gtt == NULL || !gtt->userptr) 1207 return false; 1208 1209 /* Return false if no part of the ttm_tt object lies within 1210 * the range 1211 */ 1212 size = (unsigned long)gtt->ttm.num_pages * PAGE_SIZE; 1213 if (gtt->userptr > end || gtt->userptr + size <= start) 1214 return false; 1215 1216 if (userptr) 1217 *userptr = gtt->userptr; 1218 return true; 1219 } 1220 1221 /* 1222 * amdgpu_ttm_tt_is_userptr - Have the pages backing by userptr? 1223 */ 1224 bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm) 1225 { 1226 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 1227 1228 if (gtt == NULL || !gtt->userptr) 1229 return false; 1230 1231 return true; 1232 } 1233 1234 /* 1235 * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only? 1236 */ 1237 bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm) 1238 { 1239 struct amdgpu_ttm_tt *gtt = ttm_to_amdgpu_ttm_tt(ttm); 1240 1241 if (gtt == NULL) 1242 return false; 1243 1244 return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY); 1245 } 1246 1247 /** 1248 * amdgpu_ttm_tt_pde_flags - Compute PDE flags for ttm_tt object 1249 * 1250 * @ttm: The ttm_tt object to compute the flags for 1251 * @mem: The memory registry backing this ttm_tt object 1252 * 1253 * Figure out the flags to use for a VM PDE (Page Directory Entry). 1254 */ 1255 uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem) 1256 { 1257 uint64_t flags = 0; 1258 1259 if (mem && mem->mem_type != TTM_PL_SYSTEM) 1260 flags |= AMDGPU_PTE_VALID; 1261 1262 if (mem && (mem->mem_type == TTM_PL_TT || 1263 mem->mem_type == AMDGPU_PL_PREEMPT)) { 1264 flags |= AMDGPU_PTE_SYSTEM; 1265 1266 if (ttm->caching == ttm_cached) 1267 flags |= AMDGPU_PTE_SNOOPED; 1268 } 1269 1270 if (mem && mem->mem_type == TTM_PL_VRAM && 1271 mem->bus.caching == ttm_cached) 1272 flags |= AMDGPU_PTE_SNOOPED; 1273 1274 return flags; 1275 } 1276 1277 /** 1278 * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object 1279 * 1280 * @adev: amdgpu_device pointer 1281 * @ttm: The ttm_tt object to compute the flags for 1282 * @mem: The memory registry backing this ttm_tt object 1283 * 1284 * Figure out the flags to use for a VM PTE (Page Table Entry). 1285 */ 1286 uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm, 1287 struct ttm_resource *mem) 1288 { 1289 uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem); 1290 1291 flags |= adev->gart.gart_pte_flags; 1292 flags |= AMDGPU_PTE_READABLE; 1293 1294 if (!amdgpu_ttm_tt_is_readonly(ttm)) 1295 flags |= AMDGPU_PTE_WRITEABLE; 1296 1297 return flags; 1298 } 1299 1300 /* 1301 * amdgpu_ttm_bo_eviction_valuable - Check to see if we can evict a buffer 1302 * object. 1303 * 1304 * Return true if eviction is sensible. Called by ttm_mem_evict_first() on 1305 * behalf of ttm_bo_mem_force_space() which tries to evict buffer objects until 1306 * it can find space for a new object and by ttm_bo_force_list_clean() which is 1307 * used to clean out a memory space. 1308 */ 1309 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, 1310 const struct ttm_place *place) 1311 { 1312 struct dma_resv_iter resv_cursor; 1313 struct dma_fence *f; 1314 1315 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1316 return ttm_bo_eviction_valuable(bo, place); 1317 1318 /* Swapout? */ 1319 if (bo->resource->mem_type == TTM_PL_SYSTEM) 1320 return true; 1321 1322 if (bo->type == ttm_bo_type_kernel && 1323 !amdgpu_vm_evictable(ttm_to_amdgpu_bo(bo))) 1324 return false; 1325 1326 /* If bo is a KFD BO, check if the bo belongs to the current process. 1327 * If true, then return false as any KFD process needs all its BOs to 1328 * be resident to run successfully 1329 */ 1330 dma_resv_for_each_fence(&resv_cursor, bo->base.resv, 1331 DMA_RESV_USAGE_BOOKKEEP, f) { 1332 if (amdkfd_fence_check_mm(f, current->mm)) 1333 return false; 1334 } 1335 1336 /* Preemptible BOs don't own system resources managed by the 1337 * driver (pages, VRAM, GART space). They point to resources 1338 * owned by someone else (e.g. pageable memory in user mode 1339 * or a DMABuf). They are used in a preemptible context so we 1340 * can guarantee no deadlocks and good QoS in case of MMU 1341 * notifiers or DMABuf move notifiers from the resource owner. 1342 */ 1343 if (bo->resource->mem_type == AMDGPU_PL_PREEMPT) 1344 return false; 1345 1346 if (bo->resource->mem_type == TTM_PL_TT && 1347 amdgpu_bo_encrypted(ttm_to_amdgpu_bo(bo))) 1348 return false; 1349 1350 return ttm_bo_eviction_valuable(bo, place); 1351 } 1352 1353 static void amdgpu_ttm_vram_mm_access(struct amdgpu_device *adev, loff_t pos, 1354 void *buf, size_t size, bool write) 1355 { 1356 while (size) { 1357 uint64_t aligned_pos = ALIGN_DOWN(pos, 4); 1358 uint64_t bytes = 4 - (pos & 0x3); 1359 uint32_t shift = (pos & 0x3) * 8; 1360 uint32_t mask = 0xffffffff << shift; 1361 uint32_t value = 0; 1362 1363 if (size < bytes) { 1364 mask &= 0xffffffff >> (bytes - size) * 8; 1365 bytes = size; 1366 } 1367 1368 if (mask != 0xffffffff) { 1369 amdgpu_device_mm_access(adev, aligned_pos, &value, 4, false); 1370 if (write) { 1371 value &= ~mask; 1372 value |= (*(uint32_t *)buf << shift) & mask; 1373 amdgpu_device_mm_access(adev, aligned_pos, &value, 4, true); 1374 } else { 1375 value = (value & mask) >> shift; 1376 memcpy(buf, &value, bytes); 1377 } 1378 } else { 1379 amdgpu_device_mm_access(adev, aligned_pos, buf, 4, write); 1380 } 1381 1382 pos += bytes; 1383 buf += bytes; 1384 size -= bytes; 1385 } 1386 } 1387 1388 static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo, 1389 unsigned long offset, void *buf, 1390 int len, int write) 1391 { 1392 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 1393 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 1394 struct amdgpu_res_cursor src_mm; 1395 struct amdgpu_job *job; 1396 struct dma_fence *fence; 1397 uint64_t src_addr, dst_addr; 1398 unsigned int num_dw; 1399 int r, idx; 1400 1401 if (len != PAGE_SIZE) 1402 return -EINVAL; 1403 1404 if (!adev->mman.sdma_access_ptr) 1405 return -EACCES; 1406 1407 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 1408 return -ENODEV; 1409 1410 if (write) 1411 memcpy(adev->mman.sdma_access_ptr, buf, len); 1412 1413 num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8); 1414 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.entity, 1415 AMDGPU_FENCE_OWNER_UNDEFINED, 1416 num_dw * 4, AMDGPU_IB_POOL_DELAYED, 1417 &job); 1418 if (r) 1419 goto out; 1420 1421 amdgpu_res_first(abo->tbo.resource, offset, len, &src_mm); 1422 src_addr = amdgpu_ttm_domain_start(adev, bo->resource->mem_type) + 1423 src_mm.start; 1424 dst_addr = amdgpu_bo_gpu_offset(adev->mman.sdma_access_bo); 1425 if (write) 1426 swap(src_addr, dst_addr); 1427 1428 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr, dst_addr, 1429 PAGE_SIZE, false); 1430 1431 amdgpu_ring_pad_ib(adev->mman.buffer_funcs_ring, &job->ibs[0]); 1432 WARN_ON(job->ibs[0].length_dw > num_dw); 1433 1434 fence = amdgpu_job_submit(job); 1435 1436 if (!dma_fence_wait_timeout(fence, false, adev->sdma_timeout)) 1437 r = -ETIMEDOUT; 1438 dma_fence_put(fence); 1439 1440 if (!(r || write)) 1441 memcpy(buf, adev->mman.sdma_access_ptr, len); 1442 out: 1443 drm_dev_exit(idx); 1444 return r; 1445 } 1446 1447 /** 1448 * amdgpu_ttm_access_memory - Read or Write memory that backs a buffer object. 1449 * 1450 * @bo: The buffer object to read/write 1451 * @offset: Offset into buffer object 1452 * @buf: Secondary buffer to write/read from 1453 * @len: Length in bytes of access 1454 * @write: true if writing 1455 * 1456 * This is used to access VRAM that backs a buffer object via MMIO 1457 * access for debugging purposes. 1458 */ 1459 static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo, 1460 unsigned long offset, void *buf, int len, 1461 int write) 1462 { 1463 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 1464 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 1465 struct amdgpu_res_cursor cursor; 1466 int ret = 0; 1467 1468 if (bo->resource->mem_type != TTM_PL_VRAM) 1469 return -EIO; 1470 1471 if (amdgpu_device_has_timeouts_enabled(adev) && 1472 !amdgpu_ttm_access_memory_sdma(bo, offset, buf, len, write)) 1473 return len; 1474 1475 amdgpu_res_first(bo->resource, offset, len, &cursor); 1476 while (cursor.remaining) { 1477 size_t count, size = cursor.size; 1478 loff_t pos = cursor.start; 1479 1480 count = amdgpu_device_aper_access(adev, pos, buf, size, write); 1481 size -= count; 1482 if (size) { 1483 /* using MM to access rest vram and handle un-aligned address */ 1484 pos += count; 1485 buf += count; 1486 amdgpu_ttm_vram_mm_access(adev, pos, buf, size, write); 1487 } 1488 1489 ret += cursor.size; 1490 buf += cursor.size; 1491 amdgpu_res_next(&cursor, cursor.size); 1492 } 1493 1494 return ret; 1495 } 1496 1497 static void 1498 amdgpu_bo_delete_mem_notify(struct ttm_buffer_object *bo) 1499 { 1500 amdgpu_bo_move_notify(bo, false, NULL); 1501 } 1502 1503 static struct ttm_device_funcs amdgpu_bo_driver = { 1504 .ttm_tt_create = &amdgpu_ttm_tt_create, 1505 .ttm_tt_populate = &amdgpu_ttm_tt_populate, 1506 .ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate, 1507 .ttm_tt_destroy = &amdgpu_ttm_backend_destroy, 1508 .eviction_valuable = amdgpu_ttm_bo_eviction_valuable, 1509 .evict_flags = &amdgpu_evict_flags, 1510 .move = &amdgpu_bo_move, 1511 .delete_mem_notify = &amdgpu_bo_delete_mem_notify, 1512 .release_notify = &amdgpu_bo_release_notify, 1513 .io_mem_reserve = &amdgpu_ttm_io_mem_reserve, 1514 .io_mem_pfn = amdgpu_ttm_io_mem_pfn, 1515 .access_memory = &amdgpu_ttm_access_memory, 1516 }; 1517 1518 /* 1519 * Firmware Reservation functions 1520 */ 1521 /** 1522 * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram 1523 * 1524 * @adev: amdgpu_device pointer 1525 * 1526 * free fw reserved vram if it has been reserved. 1527 */ 1528 static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev) 1529 { 1530 amdgpu_bo_free_kernel(&adev->mman.fw_vram_usage_reserved_bo, 1531 NULL, &adev->mman.fw_vram_usage_va); 1532 } 1533 1534 /* 1535 * Driver Reservation functions 1536 */ 1537 /** 1538 * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram 1539 * 1540 * @adev: amdgpu_device pointer 1541 * 1542 * free drv reserved vram if it has been reserved. 1543 */ 1544 static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev) 1545 { 1546 amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo, 1547 NULL, 1548 &adev->mman.drv_vram_usage_va); 1549 } 1550 1551 /** 1552 * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw 1553 * 1554 * @adev: amdgpu_device pointer 1555 * 1556 * create bo vram reservation from fw. 1557 */ 1558 static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev) 1559 { 1560 uint64_t vram_size = adev->gmc.visible_vram_size; 1561 1562 adev->mman.fw_vram_usage_va = NULL; 1563 adev->mman.fw_vram_usage_reserved_bo = NULL; 1564 1565 if (adev->mman.fw_vram_usage_size == 0 || 1566 adev->mman.fw_vram_usage_size > vram_size) 1567 return 0; 1568 1569 return amdgpu_bo_create_kernel_at(adev, 1570 adev->mman.fw_vram_usage_start_offset, 1571 adev->mman.fw_vram_usage_size, 1572 AMDGPU_GEM_DOMAIN_VRAM, 1573 &adev->mman.fw_vram_usage_reserved_bo, 1574 &adev->mman.fw_vram_usage_va); 1575 } 1576 1577 /** 1578 * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver 1579 * 1580 * @adev: amdgpu_device pointer 1581 * 1582 * create bo vram reservation from drv. 1583 */ 1584 static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev) 1585 { 1586 u64 vram_size = adev->gmc.visible_vram_size; 1587 1588 adev->mman.drv_vram_usage_va = NULL; 1589 adev->mman.drv_vram_usage_reserved_bo = NULL; 1590 1591 if (adev->mman.drv_vram_usage_size == 0 || 1592 adev->mman.drv_vram_usage_size > vram_size) 1593 return 0; 1594 1595 return amdgpu_bo_create_kernel_at(adev, 1596 adev->mman.drv_vram_usage_start_offset, 1597 adev->mman.drv_vram_usage_size, 1598 AMDGPU_GEM_DOMAIN_VRAM, 1599 &adev->mman.drv_vram_usage_reserved_bo, 1600 &adev->mman.drv_vram_usage_va); 1601 } 1602 1603 /* 1604 * Memoy training reservation functions 1605 */ 1606 1607 /** 1608 * amdgpu_ttm_training_reserve_vram_fini - free memory training reserved vram 1609 * 1610 * @adev: amdgpu_device pointer 1611 * 1612 * free memory training reserved vram if it has been reserved. 1613 */ 1614 static int amdgpu_ttm_training_reserve_vram_fini(struct amdgpu_device *adev) 1615 { 1616 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; 1617 1618 ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT; 1619 amdgpu_bo_free_kernel(&ctx->c2p_bo, NULL, NULL); 1620 ctx->c2p_bo = NULL; 1621 1622 return 0; 1623 } 1624 1625 static void amdgpu_ttm_training_data_block_init(struct amdgpu_device *adev) 1626 { 1627 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; 1628 1629 memset(ctx, 0, sizeof(*ctx)); 1630 1631 ctx->c2p_train_data_offset = 1632 ALIGN((adev->gmc.mc_vram_size - adev->mman.discovery_tmr_size - SZ_1M), SZ_1M); 1633 ctx->p2c_train_data_offset = 1634 (adev->gmc.mc_vram_size - GDDR6_MEM_TRAINING_OFFSET); 1635 ctx->train_data_size = 1636 GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES; 1637 1638 DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n", 1639 ctx->train_data_size, 1640 ctx->p2c_train_data_offset, 1641 ctx->c2p_train_data_offset); 1642 } 1643 1644 /* 1645 * reserve TMR memory at the top of VRAM which holds 1646 * IP Discovery data and is protected by PSP. 1647 */ 1648 static int amdgpu_ttm_reserve_tmr(struct amdgpu_device *adev) 1649 { 1650 int ret; 1651 struct psp_memory_training_context *ctx = &adev->psp.mem_train_ctx; 1652 bool mem_train_support = false; 1653 1654 if (!amdgpu_sriov_vf(adev)) { 1655 if (amdgpu_atomfirmware_mem_training_supported(adev)) 1656 mem_train_support = true; 1657 else 1658 DRM_DEBUG("memory training does not support!\n"); 1659 } 1660 1661 /* 1662 * Query reserved tmr size through atom firmwareinfo for Sienna_Cichlid and onwards for all 1663 * the use cases (IP discovery/G6 memory training/profiling/diagnostic data.etc) 1664 * 1665 * Otherwise, fallback to legacy approach to check and reserve tmr block for ip 1666 * discovery data and G6 memory training data respectively 1667 */ 1668 adev->mman.discovery_tmr_size = 1669 amdgpu_atomfirmware_get_fw_reserved_fb_size(adev); 1670 if (!adev->mman.discovery_tmr_size) 1671 adev->mman.discovery_tmr_size = DISCOVERY_TMR_OFFSET; 1672 1673 if (mem_train_support) { 1674 /* reserve vram for mem train according to TMR location */ 1675 amdgpu_ttm_training_data_block_init(adev); 1676 ret = amdgpu_bo_create_kernel_at(adev, 1677 ctx->c2p_train_data_offset, 1678 ctx->train_data_size, 1679 AMDGPU_GEM_DOMAIN_VRAM, 1680 &ctx->c2p_bo, 1681 NULL); 1682 if (ret) { 1683 DRM_ERROR("alloc c2p_bo failed(%d)!\n", ret); 1684 amdgpu_ttm_training_reserve_vram_fini(adev); 1685 return ret; 1686 } 1687 ctx->init = PSP_MEM_TRAIN_RESERVE_SUCCESS; 1688 } 1689 1690 ret = amdgpu_bo_create_kernel_at(adev, 1691 adev->gmc.real_vram_size - adev->mman.discovery_tmr_size, 1692 adev->mman.discovery_tmr_size, 1693 AMDGPU_GEM_DOMAIN_VRAM, 1694 &adev->mman.discovery_memory, 1695 NULL); 1696 if (ret) { 1697 DRM_ERROR("alloc tmr failed(%d)!\n", ret); 1698 amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); 1699 return ret; 1700 } 1701 1702 return 0; 1703 } 1704 1705 /* 1706 * amdgpu_ttm_init - Init the memory management (ttm) as well as various 1707 * gtt/vram related fields. 1708 * 1709 * This initializes all of the memory space pools that the TTM layer 1710 * will need such as the GTT space (system memory mapped to the device), 1711 * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which 1712 * can be mapped per VMID. 1713 */ 1714 int amdgpu_ttm_init(struct amdgpu_device *adev) 1715 { 1716 uint64_t gtt_size; 1717 int r; 1718 u64 vis_vram_limit; 1719 1720 mutex_init(&adev->mman.gtt_window_lock); 1721 1722 /* No others user of address space so set it to 0 */ 1723 r = ttm_device_init(&adev->mman.bdev, &amdgpu_bo_driver, adev->dev, 1724 adev_to_drm(adev)->anon_inode->i_mapping, 1725 adev_to_drm(adev)->vma_offset_manager, 1726 adev->need_swiotlb, 1727 dma_addressing_limited(adev->dev)); 1728 if (r) { 1729 DRM_ERROR("failed initializing buffer object driver(%d).\n", r); 1730 return r; 1731 } 1732 adev->mman.initialized = true; 1733 1734 /* Initialize VRAM pool with all of VRAM divided into pages */ 1735 r = amdgpu_vram_mgr_init(adev); 1736 if (r) { 1737 DRM_ERROR("Failed initializing VRAM heap.\n"); 1738 return r; 1739 } 1740 1741 /* Reduce size of CPU-visible VRAM if requested */ 1742 vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024; 1743 if (amdgpu_vis_vram_limit > 0 && 1744 vis_vram_limit <= adev->gmc.visible_vram_size) 1745 adev->gmc.visible_vram_size = vis_vram_limit; 1746 1747 /* Change the size here instead of the init above so only lpfn is affected */ 1748 amdgpu_ttm_set_buffer_funcs_status(adev, false); 1749 #ifdef CONFIG_64BIT 1750 #ifdef CONFIG_X86 1751 if (adev->gmc.xgmi.connected_to_cpu) 1752 adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base, 1753 adev->gmc.visible_vram_size); 1754 1755 else 1756 #endif 1757 adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base, 1758 adev->gmc.visible_vram_size); 1759 #endif 1760 1761 /* 1762 *The reserved vram for firmware must be pinned to the specified 1763 *place on the VRAM, so reserve it early. 1764 */ 1765 r = amdgpu_ttm_fw_reserve_vram_init(adev); 1766 if (r) { 1767 return r; 1768 } 1769 1770 /* 1771 *The reserved vram for driver must be pinned to the specified 1772 *place on the VRAM, so reserve it early. 1773 */ 1774 r = amdgpu_ttm_drv_reserve_vram_init(adev); 1775 if (r) 1776 return r; 1777 1778 /* 1779 * only NAVI10 and onwards ASIC support for IP discovery. 1780 * If IP discovery enabled, a block of memory should be 1781 * reserved for IP discovey. 1782 */ 1783 if (adev->mman.discovery_bin) { 1784 r = amdgpu_ttm_reserve_tmr(adev); 1785 if (r) 1786 return r; 1787 } 1788 1789 /* allocate memory as required for VGA 1790 * This is used for VGA emulation and pre-OS scanout buffers to 1791 * avoid display artifacts while transitioning between pre-OS 1792 * and driver. */ 1793 r = amdgpu_bo_create_kernel_at(adev, 0, adev->mman.stolen_vga_size, 1794 AMDGPU_GEM_DOMAIN_VRAM, 1795 &adev->mman.stolen_vga_memory, 1796 NULL); 1797 if (r) 1798 return r; 1799 r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_vga_size, 1800 adev->mman.stolen_extended_size, 1801 AMDGPU_GEM_DOMAIN_VRAM, 1802 &adev->mman.stolen_extended_memory, 1803 NULL); 1804 if (r) 1805 return r; 1806 r = amdgpu_bo_create_kernel_at(adev, adev->mman.stolen_reserved_offset, 1807 adev->mman.stolen_reserved_size, 1808 AMDGPU_GEM_DOMAIN_VRAM, 1809 &adev->mman.stolen_reserved_memory, 1810 NULL); 1811 if (r) 1812 return r; 1813 1814 DRM_INFO("amdgpu: %uM of VRAM memory ready\n", 1815 (unsigned) (adev->gmc.real_vram_size / (1024 * 1024))); 1816 1817 /* Compute GTT size, either based on 1/2 the size of RAM size 1818 * or whatever the user passed on module init */ 1819 if (amdgpu_gtt_size == -1) { 1820 struct sysinfo si; 1821 1822 si_meminfo(&si); 1823 /* Certain GL unit tests for large textures can cause problems 1824 * with the OOM killer since there is no way to link this memory 1825 * to a process. This was originally mitigated (but not necessarily 1826 * eliminated) by limiting the GTT size. The problem is this limit 1827 * is often too low for many modern games so just make the limit 1/2 1828 * of system memory which aligns with TTM. The OOM accounting needs 1829 * to be addressed, but we shouldn't prevent common 3D applications 1830 * from being usable just to potentially mitigate that corner case. 1831 */ 1832 gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20), 1833 (u64)si.totalram * si.mem_unit / 2); 1834 } else { 1835 gtt_size = (uint64_t)amdgpu_gtt_size << 20; 1836 } 1837 1838 /* Initialize GTT memory pool */ 1839 r = amdgpu_gtt_mgr_init(adev, gtt_size); 1840 if (r) { 1841 DRM_ERROR("Failed initializing GTT heap.\n"); 1842 return r; 1843 } 1844 DRM_INFO("amdgpu: %uM of GTT memory ready.\n", 1845 (unsigned)(gtt_size / (1024 * 1024))); 1846 1847 /* Initialize preemptible memory pool */ 1848 r = amdgpu_preempt_mgr_init(adev); 1849 if (r) { 1850 DRM_ERROR("Failed initializing PREEMPT heap.\n"); 1851 return r; 1852 } 1853 1854 /* Initialize various on-chip memory pools */ 1855 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GDS, adev->gds.gds_size); 1856 if (r) { 1857 DRM_ERROR("Failed initializing GDS heap.\n"); 1858 return r; 1859 } 1860 1861 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GWS, adev->gds.gws_size); 1862 if (r) { 1863 DRM_ERROR("Failed initializing gws heap.\n"); 1864 return r; 1865 } 1866 1867 r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_OA, adev->gds.oa_size); 1868 if (r) { 1869 DRM_ERROR("Failed initializing oa heap.\n"); 1870 return r; 1871 } 1872 1873 if (amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE, 1874 AMDGPU_GEM_DOMAIN_GTT, 1875 &adev->mman.sdma_access_bo, NULL, 1876 &adev->mman.sdma_access_ptr)) 1877 DRM_WARN("Debug VRAM access will use slowpath MM access\n"); 1878 1879 return 0; 1880 } 1881 1882 /* 1883 * amdgpu_ttm_fini - De-initialize the TTM memory pools 1884 */ 1885 void amdgpu_ttm_fini(struct amdgpu_device *adev) 1886 { 1887 int idx; 1888 if (!adev->mman.initialized) 1889 return; 1890 1891 amdgpu_ttm_training_reserve_vram_fini(adev); 1892 /* return the stolen vga memory back to VRAM */ 1893 amdgpu_bo_free_kernel(&adev->mman.stolen_vga_memory, NULL, NULL); 1894 amdgpu_bo_free_kernel(&adev->mman.stolen_extended_memory, NULL, NULL); 1895 /* return the IP Discovery TMR memory back to VRAM */ 1896 amdgpu_bo_free_kernel(&adev->mman.discovery_memory, NULL, NULL); 1897 if (adev->mman.stolen_reserved_size) 1898 amdgpu_bo_free_kernel(&adev->mman.stolen_reserved_memory, 1899 NULL, NULL); 1900 amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL, 1901 &adev->mman.sdma_access_ptr); 1902 amdgpu_ttm_fw_reserve_vram_fini(adev); 1903 amdgpu_ttm_drv_reserve_vram_fini(adev); 1904 1905 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 1906 1907 if (adev->mman.aper_base_kaddr) 1908 iounmap(adev->mman.aper_base_kaddr); 1909 adev->mman.aper_base_kaddr = NULL; 1910 1911 drm_dev_exit(idx); 1912 } 1913 1914 amdgpu_vram_mgr_fini(adev); 1915 amdgpu_gtt_mgr_fini(adev); 1916 amdgpu_preempt_mgr_fini(adev); 1917 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS); 1918 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GWS); 1919 ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA); 1920 ttm_device_fini(&adev->mman.bdev); 1921 adev->mman.initialized = false; 1922 DRM_INFO("amdgpu: ttm finalized\n"); 1923 } 1924 1925 /** 1926 * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions 1927 * 1928 * @adev: amdgpu_device pointer 1929 * @enable: true when we can use buffer functions. 1930 * 1931 * Enable/disable use of buffer functions during suspend/resume. This should 1932 * only be called at bootup or when userspace isn't running. 1933 */ 1934 void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable) 1935 { 1936 struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 1937 uint64_t size; 1938 int r; 1939 1940 if (!adev->mman.initialized || amdgpu_in_reset(adev) || 1941 adev->mman.buffer_funcs_enabled == enable) 1942 return; 1943 1944 if (enable) { 1945 struct amdgpu_ring *ring; 1946 struct drm_gpu_scheduler *sched; 1947 1948 ring = adev->mman.buffer_funcs_ring; 1949 sched = &ring->sched; 1950 r = drm_sched_entity_init(&adev->mman.entity, 1951 DRM_SCHED_PRIORITY_KERNEL, &sched, 1952 1, NULL); 1953 if (r) { 1954 DRM_ERROR("Failed setting up TTM BO move entity (%d)\n", 1955 r); 1956 return; 1957 } 1958 } else { 1959 drm_sched_entity_destroy(&adev->mman.entity); 1960 dma_fence_put(man->move); 1961 man->move = NULL; 1962 } 1963 1964 /* this just adjusts TTM size idea, which sets lpfn to the correct value */ 1965 if (enable) 1966 size = adev->gmc.real_vram_size; 1967 else 1968 size = adev->gmc.visible_vram_size; 1969 man->size = size; 1970 adev->mman.buffer_funcs_enabled = enable; 1971 } 1972 1973 static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev, 1974 bool direct_submit, 1975 unsigned int num_dw, 1976 struct dma_resv *resv, 1977 bool vm_needs_flush, 1978 struct amdgpu_job **job) 1979 { 1980 enum amdgpu_ib_pool_type pool = direct_submit ? 1981 AMDGPU_IB_POOL_DIRECT : 1982 AMDGPU_IB_POOL_DELAYED; 1983 int r; 1984 1985 r = amdgpu_job_alloc_with_ib(adev, &adev->mman.entity, 1986 AMDGPU_FENCE_OWNER_UNDEFINED, 1987 num_dw * 4, pool, job); 1988 if (r) 1989 return r; 1990 1991 if (vm_needs_flush) { 1992 (*job)->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo ? 1993 adev->gmc.pdb0_bo : 1994 adev->gart.bo); 1995 (*job)->vm_needs_flush = true; 1996 } 1997 if (!resv) 1998 return 0; 1999 2000 return drm_sched_job_add_resv_dependencies(&(*job)->base, resv, 2001 DMA_RESV_USAGE_BOOKKEEP); 2002 } 2003 2004 int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset, 2005 uint64_t dst_offset, uint32_t byte_count, 2006 struct dma_resv *resv, 2007 struct dma_fence **fence, bool direct_submit, 2008 bool vm_needs_flush, bool tmz) 2009 { 2010 struct amdgpu_device *adev = ring->adev; 2011 unsigned num_loops, num_dw; 2012 struct amdgpu_job *job; 2013 uint32_t max_bytes; 2014 unsigned i; 2015 int r; 2016 2017 if (!direct_submit && !ring->sched.ready) { 2018 DRM_ERROR("Trying to move memory with ring turned off.\n"); 2019 return -EINVAL; 2020 } 2021 2022 max_bytes = adev->mman.buffer_funcs->copy_max_bytes; 2023 num_loops = DIV_ROUND_UP(byte_count, max_bytes); 2024 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->copy_num_dw, 8); 2025 r = amdgpu_ttm_prepare_job(adev, direct_submit, num_dw, 2026 resv, vm_needs_flush, &job); 2027 if (r) 2028 return r; 2029 2030 for (i = 0; i < num_loops; i++) { 2031 uint32_t cur_size_in_bytes = min(byte_count, max_bytes); 2032 2033 amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset, 2034 dst_offset, cur_size_in_bytes, tmz); 2035 2036 src_offset += cur_size_in_bytes; 2037 dst_offset += cur_size_in_bytes; 2038 byte_count -= cur_size_in_bytes; 2039 } 2040 2041 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 2042 WARN_ON(job->ibs[0].length_dw > num_dw); 2043 if (direct_submit) 2044 r = amdgpu_job_submit_direct(job, ring, fence); 2045 else 2046 *fence = amdgpu_job_submit(job); 2047 if (r) 2048 goto error_free; 2049 2050 return r; 2051 2052 error_free: 2053 amdgpu_job_free(job); 2054 DRM_ERROR("Error scheduling IBs (%d)\n", r); 2055 return r; 2056 } 2057 2058 static int amdgpu_ttm_fill_mem(struct amdgpu_ring *ring, uint32_t src_data, 2059 uint64_t dst_addr, uint32_t byte_count, 2060 struct dma_resv *resv, 2061 struct dma_fence **fence, 2062 bool vm_needs_flush) 2063 { 2064 struct amdgpu_device *adev = ring->adev; 2065 unsigned int num_loops, num_dw; 2066 struct amdgpu_job *job; 2067 uint32_t max_bytes; 2068 unsigned int i; 2069 int r; 2070 2071 max_bytes = adev->mman.buffer_funcs->fill_max_bytes; 2072 num_loops = DIV_ROUND_UP_ULL(byte_count, max_bytes); 2073 num_dw = ALIGN(num_loops * adev->mman.buffer_funcs->fill_num_dw, 8); 2074 r = amdgpu_ttm_prepare_job(adev, false, num_dw, resv, vm_needs_flush, 2075 &job); 2076 if (r) 2077 return r; 2078 2079 for (i = 0; i < num_loops; i++) { 2080 uint32_t cur_size = min(byte_count, max_bytes); 2081 2082 amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data, dst_addr, 2083 cur_size); 2084 2085 dst_addr += cur_size; 2086 byte_count -= cur_size; 2087 } 2088 2089 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 2090 WARN_ON(job->ibs[0].length_dw > num_dw); 2091 *fence = amdgpu_job_submit(job); 2092 return 0; 2093 } 2094 2095 int amdgpu_fill_buffer(struct amdgpu_bo *bo, 2096 uint32_t src_data, 2097 struct dma_resv *resv, 2098 struct dma_fence **f) 2099 { 2100 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 2101 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; 2102 struct dma_fence *fence = NULL; 2103 struct amdgpu_res_cursor dst; 2104 int r; 2105 2106 if (!adev->mman.buffer_funcs_enabled) { 2107 DRM_ERROR("Trying to clear memory with ring turned off.\n"); 2108 return -EINVAL; 2109 } 2110 2111 amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &dst); 2112 2113 mutex_lock(&adev->mman.gtt_window_lock); 2114 while (dst.remaining) { 2115 struct dma_fence *next; 2116 uint64_t cur_size, to; 2117 2118 /* Never fill more than 256MiB at once to avoid timeouts */ 2119 cur_size = min(dst.size, 256ULL << 20); 2120 2121 r = amdgpu_ttm_map_buffer(&bo->tbo, bo->tbo.resource, &dst, 2122 1, ring, false, &cur_size, &to); 2123 if (r) 2124 goto error; 2125 2126 r = amdgpu_ttm_fill_mem(ring, src_data, to, cur_size, resv, 2127 &next, true); 2128 if (r) 2129 goto error; 2130 2131 dma_fence_put(fence); 2132 fence = next; 2133 2134 amdgpu_res_next(&dst, cur_size); 2135 } 2136 error: 2137 mutex_unlock(&adev->mman.gtt_window_lock); 2138 if (f) 2139 *f = dma_fence_get(fence); 2140 dma_fence_put(fence); 2141 return r; 2142 } 2143 2144 /** 2145 * amdgpu_ttm_evict_resources - evict memory buffers 2146 * @adev: amdgpu device object 2147 * @mem_type: evicted BO's memory type 2148 * 2149 * Evicts all @mem_type buffers on the lru list of the memory type. 2150 * 2151 * Returns: 2152 * 0 for success or a negative error code on failure. 2153 */ 2154 int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type) 2155 { 2156 struct ttm_resource_manager *man; 2157 2158 switch (mem_type) { 2159 case TTM_PL_VRAM: 2160 case TTM_PL_TT: 2161 case AMDGPU_PL_GWS: 2162 case AMDGPU_PL_GDS: 2163 case AMDGPU_PL_OA: 2164 man = ttm_manager_type(&adev->mman.bdev, mem_type); 2165 break; 2166 default: 2167 DRM_ERROR("Trying to evict invalid memory type\n"); 2168 return -EINVAL; 2169 } 2170 2171 return ttm_resource_manager_evict_all(&adev->mman.bdev, man); 2172 } 2173 2174 #if defined(CONFIG_DEBUG_FS) 2175 2176 static int amdgpu_ttm_page_pool_show(struct seq_file *m, void *unused) 2177 { 2178 struct amdgpu_device *adev = (struct amdgpu_device *)m->private; 2179 2180 return ttm_pool_debugfs(&adev->mman.bdev.pool, m); 2181 } 2182 2183 DEFINE_SHOW_ATTRIBUTE(amdgpu_ttm_page_pool); 2184 2185 /* 2186 * amdgpu_ttm_vram_read - Linear read access to VRAM 2187 * 2188 * Accesses VRAM via MMIO for debugging purposes. 2189 */ 2190 static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf, 2191 size_t size, loff_t *pos) 2192 { 2193 struct amdgpu_device *adev = file_inode(f)->i_private; 2194 ssize_t result = 0; 2195 2196 if (size & 0x3 || *pos & 0x3) 2197 return -EINVAL; 2198 2199 if (*pos >= adev->gmc.mc_vram_size) 2200 return -ENXIO; 2201 2202 size = min(size, (size_t)(adev->gmc.mc_vram_size - *pos)); 2203 while (size) { 2204 size_t bytes = min(size, AMDGPU_TTM_VRAM_MAX_DW_READ * 4); 2205 uint32_t value[AMDGPU_TTM_VRAM_MAX_DW_READ]; 2206 2207 amdgpu_device_vram_access(adev, *pos, value, bytes, false); 2208 if (copy_to_user(buf, value, bytes)) 2209 return -EFAULT; 2210 2211 result += bytes; 2212 buf += bytes; 2213 *pos += bytes; 2214 size -= bytes; 2215 } 2216 2217 return result; 2218 } 2219 2220 /* 2221 * amdgpu_ttm_vram_write - Linear write access to VRAM 2222 * 2223 * Accesses VRAM via MMIO for debugging purposes. 2224 */ 2225 static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf, 2226 size_t size, loff_t *pos) 2227 { 2228 struct amdgpu_device *adev = file_inode(f)->i_private; 2229 ssize_t result = 0; 2230 int r; 2231 2232 if (size & 0x3 || *pos & 0x3) 2233 return -EINVAL; 2234 2235 if (*pos >= adev->gmc.mc_vram_size) 2236 return -ENXIO; 2237 2238 while (size) { 2239 uint32_t value; 2240 2241 if (*pos >= adev->gmc.mc_vram_size) 2242 return result; 2243 2244 r = get_user(value, (uint32_t *)buf); 2245 if (r) 2246 return r; 2247 2248 amdgpu_device_mm_access(adev, *pos, &value, 4, true); 2249 2250 result += 4; 2251 buf += 4; 2252 *pos += 4; 2253 size -= 4; 2254 } 2255 2256 return result; 2257 } 2258 2259 static const struct file_operations amdgpu_ttm_vram_fops = { 2260 .owner = THIS_MODULE, 2261 .read = amdgpu_ttm_vram_read, 2262 .write = amdgpu_ttm_vram_write, 2263 .llseek = default_llseek, 2264 }; 2265 2266 /* 2267 * amdgpu_iomem_read - Virtual read access to GPU mapped memory 2268 * 2269 * This function is used to read memory that has been mapped to the 2270 * GPU and the known addresses are not physical addresses but instead 2271 * bus addresses (e.g., what you'd put in an IB or ring buffer). 2272 */ 2273 static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf, 2274 size_t size, loff_t *pos) 2275 { 2276 struct amdgpu_device *adev = file_inode(f)->i_private; 2277 struct iommu_domain *dom; 2278 ssize_t result = 0; 2279 int r; 2280 2281 /* retrieve the IOMMU domain if any for this device */ 2282 dom = iommu_get_domain_for_dev(adev->dev); 2283 2284 while (size) { 2285 phys_addr_t addr = *pos & PAGE_MASK; 2286 loff_t off = *pos & ~PAGE_MASK; 2287 size_t bytes = PAGE_SIZE - off; 2288 unsigned long pfn; 2289 struct page *p; 2290 void *ptr; 2291 2292 bytes = bytes < size ? bytes : size; 2293 2294 /* Translate the bus address to a physical address. If 2295 * the domain is NULL it means there is no IOMMU active 2296 * and the address translation is the identity 2297 */ 2298 addr = dom ? iommu_iova_to_phys(dom, addr) : addr; 2299 2300 pfn = addr >> PAGE_SHIFT; 2301 if (!pfn_valid(pfn)) 2302 return -EPERM; 2303 2304 p = pfn_to_page(pfn); 2305 if (p->mapping != adev->mman.bdev.dev_mapping) 2306 return -EPERM; 2307 2308 ptr = kmap_local_page(p); 2309 r = copy_to_user(buf, ptr + off, bytes); 2310 kunmap_local(ptr); 2311 if (r) 2312 return -EFAULT; 2313 2314 size -= bytes; 2315 *pos += bytes; 2316 result += bytes; 2317 } 2318 2319 return result; 2320 } 2321 2322 /* 2323 * amdgpu_iomem_write - Virtual write access to GPU mapped memory 2324 * 2325 * This function is used to write memory that has been mapped to the 2326 * GPU and the known addresses are not physical addresses but instead 2327 * bus addresses (e.g., what you'd put in an IB or ring buffer). 2328 */ 2329 static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf, 2330 size_t size, loff_t *pos) 2331 { 2332 struct amdgpu_device *adev = file_inode(f)->i_private; 2333 struct iommu_domain *dom; 2334 ssize_t result = 0; 2335 int r; 2336 2337 dom = iommu_get_domain_for_dev(adev->dev); 2338 2339 while (size) { 2340 phys_addr_t addr = *pos & PAGE_MASK; 2341 loff_t off = *pos & ~PAGE_MASK; 2342 size_t bytes = PAGE_SIZE - off; 2343 unsigned long pfn; 2344 struct page *p; 2345 void *ptr; 2346 2347 bytes = bytes < size ? bytes : size; 2348 2349 addr = dom ? iommu_iova_to_phys(dom, addr) : addr; 2350 2351 pfn = addr >> PAGE_SHIFT; 2352 if (!pfn_valid(pfn)) 2353 return -EPERM; 2354 2355 p = pfn_to_page(pfn); 2356 if (p->mapping != adev->mman.bdev.dev_mapping) 2357 return -EPERM; 2358 2359 ptr = kmap_local_page(p); 2360 r = copy_from_user(ptr + off, buf, bytes); 2361 kunmap_local(ptr); 2362 if (r) 2363 return -EFAULT; 2364 2365 size -= bytes; 2366 *pos += bytes; 2367 result += bytes; 2368 } 2369 2370 return result; 2371 } 2372 2373 static const struct file_operations amdgpu_ttm_iomem_fops = { 2374 .owner = THIS_MODULE, 2375 .read = amdgpu_iomem_read, 2376 .write = amdgpu_iomem_write, 2377 .llseek = default_llseek 2378 }; 2379 2380 #endif 2381 2382 void amdgpu_ttm_debugfs_init(struct amdgpu_device *adev) 2383 { 2384 #if defined(CONFIG_DEBUG_FS) 2385 struct drm_minor *minor = adev_to_drm(adev)->primary; 2386 struct dentry *root = minor->debugfs_root; 2387 2388 debugfs_create_file_size("amdgpu_vram", 0444, root, adev, 2389 &amdgpu_ttm_vram_fops, adev->gmc.mc_vram_size); 2390 debugfs_create_file("amdgpu_iomem", 0444, root, adev, 2391 &amdgpu_ttm_iomem_fops); 2392 debugfs_create_file("ttm_page_pool", 0444, root, adev, 2393 &amdgpu_ttm_page_pool_fops); 2394 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, 2395 TTM_PL_VRAM), 2396 root, "amdgpu_vram_mm"); 2397 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, 2398 TTM_PL_TT), 2399 root, "amdgpu_gtt_mm"); 2400 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, 2401 AMDGPU_PL_GDS), 2402 root, "amdgpu_gds_mm"); 2403 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, 2404 AMDGPU_PL_GWS), 2405 root, "amdgpu_gws_mm"); 2406 ttm_resource_manager_create_debugfs(ttm_manager_type(&adev->mman.bdev, 2407 AMDGPU_PL_OA), 2408 root, "amdgpu_oa_mm"); 2409 2410 #endif 2411 } 2412