1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2014-2018 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 #include <linux/dma-buf.h> 24 #include <linux/list.h> 25 #include <linux/pagemap.h> 26 #include <linux/sched/mm.h> 27 #include <linux/sched/task.h> 28 29 #include "amdgpu_object.h" 30 #include "amdgpu_gem.h" 31 #include "amdgpu_vm.h" 32 #include "amdgpu_amdkfd.h" 33 #include "amdgpu_dma_buf.h" 34 #include <uapi/linux/kfd_ioctl.h> 35 #include "amdgpu_xgmi.h" 36 #include "kfd_smi_events.h" 37 38 /* Userptr restore delay, just long enough to allow consecutive VM 39 * changes to accumulate 40 */ 41 #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 42 43 /* 44 * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB 45 * BO chunk 46 */ 47 #define VRAM_AVAILABLITY_ALIGN (1 << 21) 48 49 /* Impose limit on how much memory KFD can use */ 50 static struct { 51 uint64_t max_system_mem_limit; 52 uint64_t max_ttm_mem_limit; 53 int64_t system_mem_used; 54 int64_t ttm_mem_used; 55 spinlock_t mem_limit_lock; 56 } kfd_mem_limit; 57 58 static const char * const domain_bit_to_string[] = { 59 "CPU", 60 "GTT", 61 "VRAM", 62 "GDS", 63 "GWS", 64 "OA" 65 }; 66 67 #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] 68 69 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); 70 71 static bool kfd_mem_is_attached(struct amdgpu_vm *avm, 72 struct kgd_mem *mem) 73 { 74 struct kfd_mem_attachment *entry; 75 76 list_for_each_entry(entry, &mem->attachments, list) 77 if (entry->bo_va->base.vm == avm) 78 return true; 79 80 return false; 81 } 82 83 /* Set memory usage limits. Current, limits are 84 * System (TTM + userptr) memory - 15/16th System RAM 85 * TTM memory - 3/8th System RAM 86 */ 87 void amdgpu_amdkfd_gpuvm_init_mem_limits(void) 88 { 89 struct sysinfo si; 90 uint64_t mem; 91 92 si_meminfo(&si); 93 mem = si.freeram - si.freehigh; 94 mem *= si.mem_unit; 95 96 spin_lock_init(&kfd_mem_limit.mem_limit_lock); 97 kfd_mem_limit.max_system_mem_limit = mem - (mem >> 4); 98 kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3); 99 pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", 100 (kfd_mem_limit.max_system_mem_limit >> 20), 101 (kfd_mem_limit.max_ttm_mem_limit >> 20)); 102 } 103 104 void amdgpu_amdkfd_reserve_system_mem(uint64_t size) 105 { 106 kfd_mem_limit.system_mem_used += size; 107 } 108 109 /* Estimate page table size needed to represent a given memory size 110 * 111 * With 4KB pages, we need one 8 byte PTE for each 4KB of memory 112 * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB 113 * of memory (factor 256K, >> 18). ROCm user mode tries to optimize 114 * for 2MB pages for TLB efficiency. However, small allocations and 115 * fragmented system memory still need some 4KB pages. We choose a 116 * compromise that should work in most cases without reserving too 117 * much memory for page tables unnecessarily (factor 16K, >> 14). 118 */ 119 120 #define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM) 121 122 /** 123 * amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size 124 * of buffer. 125 * 126 * @adev: Device to which allocated BO belongs to 127 * @size: Size of buffer, in bytes, encapsulated by B0. This should be 128 * equivalent to amdgpu_bo_size(BO) 129 * @alloc_flag: Flag used in allocating a BO as noted above 130 * 131 * Return: returns -ENOMEM in case of error, ZERO otherwise 132 */ 133 int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, 134 uint64_t size, u32 alloc_flag) 135 { 136 uint64_t reserved_for_pt = 137 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); 138 size_t system_mem_needed, ttm_mem_needed, vram_needed; 139 int ret = 0; 140 141 system_mem_needed = 0; 142 ttm_mem_needed = 0; 143 vram_needed = 0; 144 if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { 145 system_mem_needed = size; 146 ttm_mem_needed = size; 147 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 148 /* 149 * Conservatively round up the allocation requirement to 2 MB 150 * to avoid fragmentation caused by 4K allocations in the tail 151 * 2M BO chunk. 152 */ 153 vram_needed = size; 154 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 155 system_mem_needed = size; 156 } else if (!(alloc_flag & 157 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 158 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { 159 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); 160 return -ENOMEM; 161 } 162 163 spin_lock(&kfd_mem_limit.mem_limit_lock); 164 165 if (kfd_mem_limit.system_mem_used + system_mem_needed > 166 kfd_mem_limit.max_system_mem_limit) 167 pr_debug("Set no_system_mem_limit=1 if using shared memory\n"); 168 169 if ((kfd_mem_limit.system_mem_used + system_mem_needed > 170 kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) || 171 (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > 172 kfd_mem_limit.max_ttm_mem_limit) || 173 (adev && adev->kfd.vram_used + vram_needed > 174 adev->gmc.real_vram_size - 175 atomic64_read(&adev->vram_pin_size) - 176 reserved_for_pt)) { 177 ret = -ENOMEM; 178 goto release; 179 } 180 181 /* Update memory accounting by decreasing available system 182 * memory, TTM memory and GPU memory as computed above 183 */ 184 WARN_ONCE(vram_needed && !adev, 185 "adev reference can't be null when vram is used"); 186 if (adev) { 187 adev->kfd.vram_used += vram_needed; 188 adev->kfd.vram_used_aligned += ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN); 189 } 190 kfd_mem_limit.system_mem_used += system_mem_needed; 191 kfd_mem_limit.ttm_mem_used += ttm_mem_needed; 192 193 release: 194 spin_unlock(&kfd_mem_limit.mem_limit_lock); 195 return ret; 196 } 197 198 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev, 199 uint64_t size, u32 alloc_flag) 200 { 201 spin_lock(&kfd_mem_limit.mem_limit_lock); 202 203 if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { 204 kfd_mem_limit.system_mem_used -= size; 205 kfd_mem_limit.ttm_mem_used -= size; 206 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 207 WARN_ONCE(!adev, 208 "adev reference can't be null when alloc mem flags vram is set"); 209 if (adev) { 210 adev->kfd.vram_used -= size; 211 adev->kfd.vram_used_aligned -= ALIGN(size, VRAM_AVAILABLITY_ALIGN); 212 } 213 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 214 kfd_mem_limit.system_mem_used -= size; 215 } else if (!(alloc_flag & 216 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 217 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { 218 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); 219 goto release; 220 } 221 WARN_ONCE(adev && adev->kfd.vram_used < 0, 222 "KFD VRAM memory accounting unbalanced"); 223 WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, 224 "KFD TTM memory accounting unbalanced"); 225 WARN_ONCE(kfd_mem_limit.system_mem_used < 0, 226 "KFD system memory accounting unbalanced"); 227 228 release: 229 spin_unlock(&kfd_mem_limit.mem_limit_lock); 230 } 231 232 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo) 233 { 234 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 235 u32 alloc_flags = bo->kfd_bo->alloc_flags; 236 u64 size = amdgpu_bo_size(bo); 237 238 amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags); 239 240 kfree(bo->kfd_bo); 241 } 242 243 /** 244 * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information 245 * about USERPTR or DOOREBELL or MMIO BO. 246 * @adev: Device for which dmamap BO is being created 247 * @mem: BO of peer device that is being DMA mapped. Provides parameters 248 * in building the dmamap BO 249 * @bo_out: Output parameter updated with handle of dmamap BO 250 */ 251 static int 252 create_dmamap_sg_bo(struct amdgpu_device *adev, 253 struct kgd_mem *mem, struct amdgpu_bo **bo_out) 254 { 255 struct drm_gem_object *gem_obj; 256 int ret, align; 257 258 ret = amdgpu_bo_reserve(mem->bo, false); 259 if (ret) 260 return ret; 261 262 align = 1; 263 ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align, 264 AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE, 265 ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj); 266 267 amdgpu_bo_unreserve(mem->bo); 268 269 if (ret) { 270 pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret); 271 return -EINVAL; 272 } 273 274 *bo_out = gem_to_amdgpu_bo(gem_obj); 275 (*bo_out)->parent = amdgpu_bo_ref(mem->bo); 276 return ret; 277 } 278 279 /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's 280 * reservation object. 281 * 282 * @bo: [IN] Remove eviction fence(s) from this BO 283 * @ef: [IN] This eviction fence is removed if it 284 * is present in the shared list. 285 * 286 * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. 287 */ 288 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, 289 struct amdgpu_amdkfd_fence *ef) 290 { 291 struct dma_fence *replacement; 292 293 if (!ef) 294 return -EINVAL; 295 296 /* TODO: Instead of block before we should use the fence of the page 297 * table update and TLB flush here directly. 298 */ 299 replacement = dma_fence_get_stub(); 300 dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context, 301 replacement, DMA_RESV_USAGE_BOOKKEEP); 302 dma_fence_put(replacement); 303 return 0; 304 } 305 306 int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) 307 { 308 struct amdgpu_bo *root = bo; 309 struct amdgpu_vm_bo_base *vm_bo; 310 struct amdgpu_vm *vm; 311 struct amdkfd_process_info *info; 312 struct amdgpu_amdkfd_fence *ef; 313 int ret; 314 315 /* we can always get vm_bo from root PD bo.*/ 316 while (root->parent) 317 root = root->parent; 318 319 vm_bo = root->vm_bo; 320 if (!vm_bo) 321 return 0; 322 323 vm = vm_bo->vm; 324 if (!vm) 325 return 0; 326 327 info = vm->process_info; 328 if (!info || !info->eviction_fence) 329 return 0; 330 331 ef = container_of(dma_fence_get(&info->eviction_fence->base), 332 struct amdgpu_amdkfd_fence, base); 333 334 BUG_ON(!dma_resv_trylock(bo->tbo.base.resv)); 335 ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef); 336 dma_resv_unlock(bo->tbo.base.resv); 337 338 dma_fence_put(&ef->base); 339 return ret; 340 } 341 342 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, 343 bool wait) 344 { 345 struct ttm_operation_ctx ctx = { false, false }; 346 int ret; 347 348 if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), 349 "Called with userptr BO")) 350 return -EINVAL; 351 352 amdgpu_bo_placement_from_domain(bo, domain); 353 354 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 355 if (ret) 356 goto validate_fail; 357 if (wait) 358 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); 359 360 validate_fail: 361 return ret; 362 } 363 364 static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo) 365 { 366 return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false); 367 } 368 369 /* vm_validate_pt_pd_bos - Validate page table and directory BOs 370 * 371 * Page directories are not updated here because huge page handling 372 * during page table updates can invalidate page directory entries 373 * again. Page directories are only updated after updating page 374 * tables. 375 */ 376 static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm) 377 { 378 struct amdgpu_bo *pd = vm->root.bo; 379 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); 380 int ret; 381 382 ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate_vm_bo, NULL); 383 if (ret) { 384 pr_err("failed to validate PT BOs\n"); 385 return ret; 386 } 387 388 vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo); 389 390 return 0; 391 } 392 393 static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync) 394 { 395 struct amdgpu_bo *pd = vm->root.bo; 396 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); 397 int ret; 398 399 ret = amdgpu_vm_update_pdes(adev, vm, false); 400 if (ret) 401 return ret; 402 403 return amdgpu_sync_fence(sync, vm->last_update); 404 } 405 406 static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem) 407 { 408 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); 409 bool coherent = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; 410 bool uncached = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; 411 uint32_t mapping_flags; 412 uint64_t pte_flags; 413 bool snoop = false; 414 415 mapping_flags = AMDGPU_VM_PAGE_READABLE; 416 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE) 417 mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE; 418 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE) 419 mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; 420 421 switch (adev->asic_type) { 422 case CHIP_ARCTURUS: 423 case CHIP_ALDEBARAN: 424 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 425 if (bo_adev == adev) { 426 if (uncached) 427 mapping_flags |= AMDGPU_VM_MTYPE_UC; 428 else if (coherent) 429 mapping_flags |= AMDGPU_VM_MTYPE_CC; 430 else 431 mapping_flags |= AMDGPU_VM_MTYPE_RW; 432 if (adev->asic_type == CHIP_ALDEBARAN && 433 adev->gmc.xgmi.connected_to_cpu) 434 snoop = true; 435 } else { 436 if (uncached || coherent) 437 mapping_flags |= AMDGPU_VM_MTYPE_UC; 438 else 439 mapping_flags |= AMDGPU_VM_MTYPE_NC; 440 if (amdgpu_xgmi_same_hive(adev, bo_adev)) 441 snoop = true; 442 } 443 } else { 444 if (uncached || coherent) 445 mapping_flags |= AMDGPU_VM_MTYPE_UC; 446 else 447 mapping_flags |= AMDGPU_VM_MTYPE_NC; 448 snoop = true; 449 } 450 break; 451 default: 452 if (uncached || coherent) 453 mapping_flags |= AMDGPU_VM_MTYPE_UC; 454 else 455 mapping_flags |= AMDGPU_VM_MTYPE_NC; 456 457 if (!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) 458 snoop = true; 459 } 460 461 pte_flags = amdgpu_gem_va_map_flags(adev, mapping_flags); 462 pte_flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; 463 464 return pte_flags; 465 } 466 467 /** 468 * create_sg_table() - Create an sg_table for a contiguous DMA addr range 469 * @addr: The starting address to point to 470 * @size: Size of memory area in bytes being pointed to 471 * 472 * Allocates an instance of sg_table and initializes it to point to memory 473 * area specified by input parameters. The address used to build is assumed 474 * to be DMA mapped, if needed. 475 * 476 * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table 477 * because they are physically contiguous. 478 * 479 * Return: Initialized instance of SG Table or NULL 480 */ 481 static struct sg_table *create_sg_table(uint64_t addr, uint32_t size) 482 { 483 struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL); 484 485 if (!sg) 486 return NULL; 487 if (sg_alloc_table(sg, 1, GFP_KERNEL)) { 488 kfree(sg); 489 return NULL; 490 } 491 sg_dma_address(sg->sgl) = addr; 492 sg->sgl->length = size; 493 #ifdef CONFIG_NEED_SG_DMA_LENGTH 494 sg->sgl->dma_length = size; 495 #endif 496 return sg; 497 } 498 499 static int 500 kfd_mem_dmamap_userptr(struct kgd_mem *mem, 501 struct kfd_mem_attachment *attachment) 502 { 503 enum dma_data_direction direction = 504 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 505 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 506 struct ttm_operation_ctx ctx = {.interruptible = true}; 507 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 508 struct amdgpu_device *adev = attachment->adev; 509 struct ttm_tt *src_ttm = mem->bo->tbo.ttm; 510 struct ttm_tt *ttm = bo->tbo.ttm; 511 int ret; 512 513 ttm->sg = kmalloc(sizeof(*ttm->sg), GFP_KERNEL); 514 if (unlikely(!ttm->sg)) 515 return -ENOMEM; 516 517 if (WARN_ON(ttm->num_pages != src_ttm->num_pages)) 518 return -EINVAL; 519 520 /* Same sequence as in amdgpu_ttm_tt_pin_userptr */ 521 ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages, 522 ttm->num_pages, 0, 523 (u64)ttm->num_pages << PAGE_SHIFT, 524 GFP_KERNEL); 525 if (unlikely(ret)) 526 goto free_sg; 527 528 ret = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); 529 if (unlikely(ret)) 530 goto release_sg; 531 532 drm_prime_sg_to_dma_addr_array(ttm->sg, ttm->dma_address, 533 ttm->num_pages); 534 535 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 536 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 537 if (ret) 538 goto unmap_sg; 539 540 return 0; 541 542 unmap_sg: 543 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); 544 release_sg: 545 pr_err("DMA map userptr failed: %d\n", ret); 546 sg_free_table(ttm->sg); 547 free_sg: 548 kfree(ttm->sg); 549 ttm->sg = NULL; 550 return ret; 551 } 552 553 static int 554 kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment) 555 { 556 struct ttm_operation_ctx ctx = {.interruptible = true}; 557 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 558 559 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 560 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 561 } 562 563 /** 564 * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO 565 * @mem: SG BO of the DOORBELL or MMIO resource on the owning device 566 * @attachment: Virtual address attachment of the BO on accessing device 567 * 568 * An access request from the device that owns DOORBELL does not require DMA mapping. 569 * This is because the request doesn't go through PCIe root complex i.e. it instead 570 * loops back. The need to DMA map arises only when accessing peer device's DOORBELL 571 * 572 * In contrast, all access requests for MMIO need to be DMA mapped without regard to 573 * device ownership. This is because access requests for MMIO go through PCIe root 574 * complex. 575 * 576 * This is accomplished in two steps: 577 * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used 578 * in updating requesting device's page table 579 * - Signal TTM to mark memory pointed to by requesting device's BO as GPU 580 * accessible. This allows an update of requesting device's page table 581 * with entries associated with DOOREBELL or MMIO memory 582 * 583 * This method is invoked in the following contexts: 584 * - Mapping of DOORBELL or MMIO BO of same or peer device 585 * - Validating an evicted DOOREBELL or MMIO BO on device seeking access 586 * 587 * Return: ZERO if successful, NON-ZERO otherwise 588 */ 589 static int 590 kfd_mem_dmamap_sg_bo(struct kgd_mem *mem, 591 struct kfd_mem_attachment *attachment) 592 { 593 struct ttm_operation_ctx ctx = {.interruptible = true}; 594 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 595 struct amdgpu_device *adev = attachment->adev; 596 struct ttm_tt *ttm = bo->tbo.ttm; 597 enum dma_data_direction dir; 598 dma_addr_t dma_addr; 599 bool mmio; 600 int ret; 601 602 /* Expect SG Table of dmapmap BO to be NULL */ 603 mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP); 604 if (unlikely(ttm->sg)) { 605 pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio); 606 return -EINVAL; 607 } 608 609 dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 610 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 611 dma_addr = mem->bo->tbo.sg->sgl->dma_address; 612 pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length); 613 pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr); 614 dma_addr = dma_map_resource(adev->dev, dma_addr, 615 mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC); 616 ret = dma_mapping_error(adev->dev, dma_addr); 617 if (unlikely(ret)) 618 return ret; 619 pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr); 620 621 ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length); 622 if (unlikely(!ttm->sg)) { 623 ret = -ENOMEM; 624 goto unmap_sg; 625 } 626 627 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 628 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 629 if (unlikely(ret)) 630 goto free_sg; 631 632 return ret; 633 634 free_sg: 635 sg_free_table(ttm->sg); 636 kfree(ttm->sg); 637 ttm->sg = NULL; 638 unmap_sg: 639 dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length, 640 dir, DMA_ATTR_SKIP_CPU_SYNC); 641 return ret; 642 } 643 644 static int 645 kfd_mem_dmamap_attachment(struct kgd_mem *mem, 646 struct kfd_mem_attachment *attachment) 647 { 648 switch (attachment->type) { 649 case KFD_MEM_ATT_SHARED: 650 return 0; 651 case KFD_MEM_ATT_USERPTR: 652 return kfd_mem_dmamap_userptr(mem, attachment); 653 case KFD_MEM_ATT_DMABUF: 654 return kfd_mem_dmamap_dmabuf(attachment); 655 case KFD_MEM_ATT_SG: 656 return kfd_mem_dmamap_sg_bo(mem, attachment); 657 default: 658 WARN_ON_ONCE(1); 659 } 660 return -EINVAL; 661 } 662 663 static void 664 kfd_mem_dmaunmap_userptr(struct kgd_mem *mem, 665 struct kfd_mem_attachment *attachment) 666 { 667 enum dma_data_direction direction = 668 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 669 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 670 struct ttm_operation_ctx ctx = {.interruptible = false}; 671 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 672 struct amdgpu_device *adev = attachment->adev; 673 struct ttm_tt *ttm = bo->tbo.ttm; 674 675 if (unlikely(!ttm->sg)) 676 return; 677 678 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 679 ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 680 681 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); 682 sg_free_table(ttm->sg); 683 kfree(ttm->sg); 684 ttm->sg = NULL; 685 } 686 687 static void 688 kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment) 689 { 690 struct ttm_operation_ctx ctx = {.interruptible = true}; 691 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 692 693 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 694 ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 695 } 696 697 /** 698 * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO 699 * @mem: SG BO of the DOORBELL or MMIO resource on the owning device 700 * @attachment: Virtual address attachment of the BO on accessing device 701 * 702 * The method performs following steps: 703 * - Signal TTM to mark memory pointed to by BO as GPU inaccessible 704 * - Free SG Table that is used to encapsulate DMA mapped memory of 705 * peer device's DOORBELL or MMIO memory 706 * 707 * This method is invoked in the following contexts: 708 * UNMapping of DOORBELL or MMIO BO on a device having access to its memory 709 * Eviction of DOOREBELL or MMIO BO on device having access to its memory 710 * 711 * Return: void 712 */ 713 static void 714 kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem, 715 struct kfd_mem_attachment *attachment) 716 { 717 struct ttm_operation_ctx ctx = {.interruptible = true}; 718 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 719 struct amdgpu_device *adev = attachment->adev; 720 struct ttm_tt *ttm = bo->tbo.ttm; 721 enum dma_data_direction dir; 722 723 if (unlikely(!ttm->sg)) { 724 pr_err("SG Table of BO is UNEXPECTEDLY NULL"); 725 return; 726 } 727 728 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 729 ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 730 731 dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 732 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 733 dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address, 734 ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC); 735 sg_free_table(ttm->sg); 736 kfree(ttm->sg); 737 ttm->sg = NULL; 738 bo->tbo.sg = NULL; 739 } 740 741 static void 742 kfd_mem_dmaunmap_attachment(struct kgd_mem *mem, 743 struct kfd_mem_attachment *attachment) 744 { 745 switch (attachment->type) { 746 case KFD_MEM_ATT_SHARED: 747 break; 748 case KFD_MEM_ATT_USERPTR: 749 kfd_mem_dmaunmap_userptr(mem, attachment); 750 break; 751 case KFD_MEM_ATT_DMABUF: 752 kfd_mem_dmaunmap_dmabuf(attachment); 753 break; 754 case KFD_MEM_ATT_SG: 755 kfd_mem_dmaunmap_sg_bo(mem, attachment); 756 break; 757 default: 758 WARN_ON_ONCE(1); 759 } 760 } 761 762 static int 763 kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem, 764 struct amdgpu_bo **bo) 765 { 766 struct drm_gem_object *gobj; 767 int ret; 768 769 if (!mem->dmabuf) { 770 mem->dmabuf = amdgpu_gem_prime_export(&mem->bo->tbo.base, 771 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 772 DRM_RDWR : 0); 773 if (IS_ERR(mem->dmabuf)) { 774 ret = PTR_ERR(mem->dmabuf); 775 mem->dmabuf = NULL; 776 return ret; 777 } 778 } 779 780 gobj = amdgpu_gem_prime_import(adev_to_drm(adev), mem->dmabuf); 781 if (IS_ERR(gobj)) 782 return PTR_ERR(gobj); 783 784 *bo = gem_to_amdgpu_bo(gobj); 785 (*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE; 786 787 return 0; 788 } 789 790 /* kfd_mem_attach - Add a BO to a VM 791 * 792 * Everything that needs to bo done only once when a BO is first added 793 * to a VM. It can later be mapped and unmapped many times without 794 * repeating these steps. 795 * 796 * 0. Create BO for DMA mapping, if needed 797 * 1. Allocate and initialize BO VA entry data structure 798 * 2. Add BO to the VM 799 * 3. Determine ASIC-specific PTE flags 800 * 4. Alloc page tables and directories if needed 801 * 4a. Validate new page tables and directories 802 */ 803 static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem, 804 struct amdgpu_vm *vm, bool is_aql) 805 { 806 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); 807 unsigned long bo_size = mem->bo->tbo.base.size; 808 uint64_t va = mem->va; 809 struct kfd_mem_attachment *attachment[2] = {NULL, NULL}; 810 struct amdgpu_bo *bo[2] = {NULL, NULL}; 811 bool same_hive = false; 812 int i, ret; 813 814 if (!va) { 815 pr_err("Invalid VA when adding BO to VM\n"); 816 return -EINVAL; 817 } 818 819 /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices 820 * 821 * The access path of MMIO and DOORBELL BOs of is always over PCIe. 822 * In contrast the access path of VRAM BOs depens upon the type of 823 * link that connects the peer device. Access over PCIe is allowed 824 * if peer device has large BAR. In contrast, access over xGMI is 825 * allowed for both small and large BAR configurations of peer device 826 */ 827 if ((adev != bo_adev) && 828 ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) || 829 (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) || 830 (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { 831 if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM) 832 same_hive = amdgpu_xgmi_same_hive(adev, bo_adev); 833 if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev)) 834 return -EINVAL; 835 } 836 837 for (i = 0; i <= is_aql; i++) { 838 attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL); 839 if (unlikely(!attachment[i])) { 840 ret = -ENOMEM; 841 goto unwind; 842 } 843 844 pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va, 845 va + bo_size, vm); 846 847 if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) || 848 (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) || 849 same_hive) { 850 /* Mappings on the local GPU, or VRAM mappings in the 851 * local hive, or userptr mapping IOMMU direct map mode 852 * share the original BO 853 */ 854 attachment[i]->type = KFD_MEM_ATT_SHARED; 855 bo[i] = mem->bo; 856 drm_gem_object_get(&bo[i]->tbo.base); 857 } else if (i > 0) { 858 /* Multiple mappings on the same GPU share the BO */ 859 attachment[i]->type = KFD_MEM_ATT_SHARED; 860 bo[i] = bo[0]; 861 drm_gem_object_get(&bo[i]->tbo.base); 862 } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) { 863 /* Create an SG BO to DMA-map userptrs on other GPUs */ 864 attachment[i]->type = KFD_MEM_ATT_USERPTR; 865 ret = create_dmamap_sg_bo(adev, mem, &bo[i]); 866 if (ret) 867 goto unwind; 868 /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */ 869 } else if (mem->bo->tbo.type == ttm_bo_type_sg) { 870 WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL || 871 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP), 872 "Handing invalid SG BO in ATTACH request"); 873 attachment[i]->type = KFD_MEM_ATT_SG; 874 ret = create_dmamap_sg_bo(adev, mem, &bo[i]); 875 if (ret) 876 goto unwind; 877 /* Enable acces to GTT and VRAM BOs of peer devices */ 878 } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT || 879 mem->domain == AMDGPU_GEM_DOMAIN_VRAM) { 880 attachment[i]->type = KFD_MEM_ATT_DMABUF; 881 ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]); 882 if (ret) 883 goto unwind; 884 pr_debug("Employ DMABUF mechanism to enable peer GPU access\n"); 885 } else { 886 WARN_ONCE(true, "Handling invalid ATTACH request"); 887 ret = -EINVAL; 888 goto unwind; 889 } 890 891 /* Add BO to VM internal data structures */ 892 ret = amdgpu_bo_reserve(bo[i], false); 893 if (ret) { 894 pr_debug("Unable to reserve BO during memory attach"); 895 goto unwind; 896 } 897 attachment[i]->bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]); 898 amdgpu_bo_unreserve(bo[i]); 899 if (unlikely(!attachment[i]->bo_va)) { 900 ret = -ENOMEM; 901 pr_err("Failed to add BO object to VM. ret == %d\n", 902 ret); 903 goto unwind; 904 } 905 attachment[i]->va = va; 906 attachment[i]->pte_flags = get_pte_flags(adev, mem); 907 attachment[i]->adev = adev; 908 list_add(&attachment[i]->list, &mem->attachments); 909 910 va += bo_size; 911 } 912 913 return 0; 914 915 unwind: 916 for (; i >= 0; i--) { 917 if (!attachment[i]) 918 continue; 919 if (attachment[i]->bo_va) { 920 amdgpu_bo_reserve(bo[i], true); 921 amdgpu_vm_bo_del(adev, attachment[i]->bo_va); 922 amdgpu_bo_unreserve(bo[i]); 923 list_del(&attachment[i]->list); 924 } 925 if (bo[i]) 926 drm_gem_object_put(&bo[i]->tbo.base); 927 kfree(attachment[i]); 928 } 929 return ret; 930 } 931 932 static void kfd_mem_detach(struct kfd_mem_attachment *attachment) 933 { 934 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 935 936 pr_debug("\t remove VA 0x%llx in entry %p\n", 937 attachment->va, attachment); 938 amdgpu_vm_bo_del(attachment->adev, attachment->bo_va); 939 drm_gem_object_put(&bo->tbo.base); 940 list_del(&attachment->list); 941 kfree(attachment); 942 } 943 944 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem, 945 struct amdkfd_process_info *process_info, 946 bool userptr) 947 { 948 struct ttm_validate_buffer *entry = &mem->validate_list; 949 struct amdgpu_bo *bo = mem->bo; 950 951 INIT_LIST_HEAD(&entry->head); 952 entry->num_shared = 1; 953 entry->bo = &bo->tbo; 954 mutex_lock(&process_info->lock); 955 if (userptr) 956 list_add_tail(&entry->head, &process_info->userptr_valid_list); 957 else 958 list_add_tail(&entry->head, &process_info->kfd_bo_list); 959 mutex_unlock(&process_info->lock); 960 } 961 962 static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem, 963 struct amdkfd_process_info *process_info) 964 { 965 struct ttm_validate_buffer *bo_list_entry; 966 967 bo_list_entry = &mem->validate_list; 968 mutex_lock(&process_info->lock); 969 list_del(&bo_list_entry->head); 970 mutex_unlock(&process_info->lock); 971 } 972 973 /* Initializes user pages. It registers the MMU notifier and validates 974 * the userptr BO in the GTT domain. 975 * 976 * The BO must already be on the userptr_valid_list. Otherwise an 977 * eviction and restore may happen that leaves the new BO unmapped 978 * with the user mode queues running. 979 * 980 * Takes the process_info->lock to protect against concurrent restore 981 * workers. 982 * 983 * Returns 0 for success, negative errno for errors. 984 */ 985 static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr, 986 bool criu_resume) 987 { 988 struct amdkfd_process_info *process_info = mem->process_info; 989 struct amdgpu_bo *bo = mem->bo; 990 struct ttm_operation_ctx ctx = { true, false }; 991 int ret = 0; 992 993 mutex_lock(&process_info->lock); 994 995 ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0); 996 if (ret) { 997 pr_err("%s: Failed to set userptr: %d\n", __func__, ret); 998 goto out; 999 } 1000 1001 ret = amdgpu_mn_register(bo, user_addr); 1002 if (ret) { 1003 pr_err("%s: Failed to register MMU notifier: %d\n", 1004 __func__, ret); 1005 goto out; 1006 } 1007 1008 if (criu_resume) { 1009 /* 1010 * During a CRIU restore operation, the userptr buffer objects 1011 * will be validated in the restore_userptr_work worker at a 1012 * later stage when it is scheduled by another ioctl called by 1013 * CRIU master process for the target pid for restore. 1014 */ 1015 atomic_inc(&mem->invalid); 1016 mutex_unlock(&process_info->lock); 1017 return 0; 1018 } 1019 1020 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages); 1021 if (ret) { 1022 pr_err("%s: Failed to get user pages: %d\n", __func__, ret); 1023 goto unregister_out; 1024 } 1025 1026 ret = amdgpu_bo_reserve(bo, true); 1027 if (ret) { 1028 pr_err("%s: Failed to reserve BO\n", __func__); 1029 goto release_out; 1030 } 1031 amdgpu_bo_placement_from_domain(bo, mem->domain); 1032 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 1033 if (ret) 1034 pr_err("%s: failed to validate BO\n", __func__); 1035 amdgpu_bo_unreserve(bo); 1036 1037 release_out: 1038 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); 1039 unregister_out: 1040 if (ret) 1041 amdgpu_mn_unregister(bo); 1042 out: 1043 mutex_unlock(&process_info->lock); 1044 return ret; 1045 } 1046 1047 /* Reserving a BO and its page table BOs must happen atomically to 1048 * avoid deadlocks. Some operations update multiple VMs at once. Track 1049 * all the reservation info in a context structure. Optionally a sync 1050 * object can track VM updates. 1051 */ 1052 struct bo_vm_reservation_context { 1053 struct amdgpu_bo_list_entry kfd_bo; /* BO list entry for the KFD BO */ 1054 unsigned int n_vms; /* Number of VMs reserved */ 1055 struct amdgpu_bo_list_entry *vm_pd; /* Array of VM BO list entries */ 1056 struct ww_acquire_ctx ticket; /* Reservation ticket */ 1057 struct list_head list, duplicates; /* BO lists */ 1058 struct amdgpu_sync *sync; /* Pointer to sync object */ 1059 bool reserved; /* Whether BOs are reserved */ 1060 }; 1061 1062 enum bo_vm_match { 1063 BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */ 1064 BO_VM_MAPPED, /* Match VMs where a BO is mapped */ 1065 BO_VM_ALL, /* Match all VMs a BO was added to */ 1066 }; 1067 1068 /** 1069 * reserve_bo_and_vm - reserve a BO and a VM unconditionally. 1070 * @mem: KFD BO structure. 1071 * @vm: the VM to reserve. 1072 * @ctx: the struct that will be used in unreserve_bo_and_vms(). 1073 */ 1074 static int reserve_bo_and_vm(struct kgd_mem *mem, 1075 struct amdgpu_vm *vm, 1076 struct bo_vm_reservation_context *ctx) 1077 { 1078 struct amdgpu_bo *bo = mem->bo; 1079 int ret; 1080 1081 WARN_ON(!vm); 1082 1083 ctx->reserved = false; 1084 ctx->n_vms = 1; 1085 ctx->sync = &mem->sync; 1086 1087 INIT_LIST_HEAD(&ctx->list); 1088 INIT_LIST_HEAD(&ctx->duplicates); 1089 1090 ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd), GFP_KERNEL); 1091 if (!ctx->vm_pd) 1092 return -ENOMEM; 1093 1094 ctx->kfd_bo.priority = 0; 1095 ctx->kfd_bo.tv.bo = &bo->tbo; 1096 ctx->kfd_bo.tv.num_shared = 1; 1097 list_add(&ctx->kfd_bo.tv.head, &ctx->list); 1098 1099 amdgpu_vm_get_pd_bo(vm, &ctx->list, &ctx->vm_pd[0]); 1100 1101 ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list, 1102 false, &ctx->duplicates); 1103 if (ret) { 1104 pr_err("Failed to reserve buffers in ttm.\n"); 1105 kfree(ctx->vm_pd); 1106 ctx->vm_pd = NULL; 1107 return ret; 1108 } 1109 1110 ctx->reserved = true; 1111 return 0; 1112 } 1113 1114 /** 1115 * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally 1116 * @mem: KFD BO structure. 1117 * @vm: the VM to reserve. If NULL, then all VMs associated with the BO 1118 * is used. Otherwise, a single VM associated with the BO. 1119 * @map_type: the mapping status that will be used to filter the VMs. 1120 * @ctx: the struct that will be used in unreserve_bo_and_vms(). 1121 * 1122 * Returns 0 for success, negative for failure. 1123 */ 1124 static int reserve_bo_and_cond_vms(struct kgd_mem *mem, 1125 struct amdgpu_vm *vm, enum bo_vm_match map_type, 1126 struct bo_vm_reservation_context *ctx) 1127 { 1128 struct amdgpu_bo *bo = mem->bo; 1129 struct kfd_mem_attachment *entry; 1130 unsigned int i; 1131 int ret; 1132 1133 ctx->reserved = false; 1134 ctx->n_vms = 0; 1135 ctx->vm_pd = NULL; 1136 ctx->sync = &mem->sync; 1137 1138 INIT_LIST_HEAD(&ctx->list); 1139 INIT_LIST_HEAD(&ctx->duplicates); 1140 1141 list_for_each_entry(entry, &mem->attachments, list) { 1142 if ((vm && vm != entry->bo_va->base.vm) || 1143 (entry->is_mapped != map_type 1144 && map_type != BO_VM_ALL)) 1145 continue; 1146 1147 ctx->n_vms++; 1148 } 1149 1150 if (ctx->n_vms != 0) { 1151 ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd), 1152 GFP_KERNEL); 1153 if (!ctx->vm_pd) 1154 return -ENOMEM; 1155 } 1156 1157 ctx->kfd_bo.priority = 0; 1158 ctx->kfd_bo.tv.bo = &bo->tbo; 1159 ctx->kfd_bo.tv.num_shared = 1; 1160 list_add(&ctx->kfd_bo.tv.head, &ctx->list); 1161 1162 i = 0; 1163 list_for_each_entry(entry, &mem->attachments, list) { 1164 if ((vm && vm != entry->bo_va->base.vm) || 1165 (entry->is_mapped != map_type 1166 && map_type != BO_VM_ALL)) 1167 continue; 1168 1169 amdgpu_vm_get_pd_bo(entry->bo_va->base.vm, &ctx->list, 1170 &ctx->vm_pd[i]); 1171 i++; 1172 } 1173 1174 ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list, 1175 false, &ctx->duplicates); 1176 if (ret) { 1177 pr_err("Failed to reserve buffers in ttm.\n"); 1178 kfree(ctx->vm_pd); 1179 ctx->vm_pd = NULL; 1180 return ret; 1181 } 1182 1183 ctx->reserved = true; 1184 return 0; 1185 } 1186 1187 /** 1188 * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context 1189 * @ctx: Reservation context to unreserve 1190 * @wait: Optionally wait for a sync object representing pending VM updates 1191 * @intr: Whether the wait is interruptible 1192 * 1193 * Also frees any resources allocated in 1194 * reserve_bo_and_(cond_)vm(s). Returns the status from 1195 * amdgpu_sync_wait. 1196 */ 1197 static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx, 1198 bool wait, bool intr) 1199 { 1200 int ret = 0; 1201 1202 if (wait) 1203 ret = amdgpu_sync_wait(ctx->sync, intr); 1204 1205 if (ctx->reserved) 1206 ttm_eu_backoff_reservation(&ctx->ticket, &ctx->list); 1207 kfree(ctx->vm_pd); 1208 1209 ctx->sync = NULL; 1210 1211 ctx->reserved = false; 1212 ctx->vm_pd = NULL; 1213 1214 return ret; 1215 } 1216 1217 static void unmap_bo_from_gpuvm(struct kgd_mem *mem, 1218 struct kfd_mem_attachment *entry, 1219 struct amdgpu_sync *sync) 1220 { 1221 struct amdgpu_bo_va *bo_va = entry->bo_va; 1222 struct amdgpu_device *adev = entry->adev; 1223 struct amdgpu_vm *vm = bo_va->base.vm; 1224 1225 amdgpu_vm_bo_unmap(adev, bo_va, entry->va); 1226 1227 amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update); 1228 1229 amdgpu_sync_fence(sync, bo_va->last_pt_update); 1230 1231 kfd_mem_dmaunmap_attachment(mem, entry); 1232 } 1233 1234 static int update_gpuvm_pte(struct kgd_mem *mem, 1235 struct kfd_mem_attachment *entry, 1236 struct amdgpu_sync *sync) 1237 { 1238 struct amdgpu_bo_va *bo_va = entry->bo_va; 1239 struct amdgpu_device *adev = entry->adev; 1240 int ret; 1241 1242 ret = kfd_mem_dmamap_attachment(mem, entry); 1243 if (ret) 1244 return ret; 1245 1246 /* Update the page tables */ 1247 ret = amdgpu_vm_bo_update(adev, bo_va, false); 1248 if (ret) { 1249 pr_err("amdgpu_vm_bo_update failed\n"); 1250 return ret; 1251 } 1252 1253 return amdgpu_sync_fence(sync, bo_va->last_pt_update); 1254 } 1255 1256 static int map_bo_to_gpuvm(struct kgd_mem *mem, 1257 struct kfd_mem_attachment *entry, 1258 struct amdgpu_sync *sync, 1259 bool no_update_pte) 1260 { 1261 int ret; 1262 1263 /* Set virtual address for the allocation */ 1264 ret = amdgpu_vm_bo_map(entry->adev, entry->bo_va, entry->va, 0, 1265 amdgpu_bo_size(entry->bo_va->base.bo), 1266 entry->pte_flags); 1267 if (ret) { 1268 pr_err("Failed to map VA 0x%llx in vm. ret %d\n", 1269 entry->va, ret); 1270 return ret; 1271 } 1272 1273 if (no_update_pte) 1274 return 0; 1275 1276 ret = update_gpuvm_pte(mem, entry, sync); 1277 if (ret) { 1278 pr_err("update_gpuvm_pte() failed\n"); 1279 goto update_gpuvm_pte_failed; 1280 } 1281 1282 return 0; 1283 1284 update_gpuvm_pte_failed: 1285 unmap_bo_from_gpuvm(mem, entry, sync); 1286 return ret; 1287 } 1288 1289 static int process_validate_vms(struct amdkfd_process_info *process_info) 1290 { 1291 struct amdgpu_vm *peer_vm; 1292 int ret; 1293 1294 list_for_each_entry(peer_vm, &process_info->vm_list_head, 1295 vm_list_node) { 1296 ret = vm_validate_pt_pd_bos(peer_vm); 1297 if (ret) 1298 return ret; 1299 } 1300 1301 return 0; 1302 } 1303 1304 static int process_sync_pds_resv(struct amdkfd_process_info *process_info, 1305 struct amdgpu_sync *sync) 1306 { 1307 struct amdgpu_vm *peer_vm; 1308 int ret; 1309 1310 list_for_each_entry(peer_vm, &process_info->vm_list_head, 1311 vm_list_node) { 1312 struct amdgpu_bo *pd = peer_vm->root.bo; 1313 1314 ret = amdgpu_sync_resv(NULL, sync, pd->tbo.base.resv, 1315 AMDGPU_SYNC_NE_OWNER, 1316 AMDGPU_FENCE_OWNER_KFD); 1317 if (ret) 1318 return ret; 1319 } 1320 1321 return 0; 1322 } 1323 1324 static int process_update_pds(struct amdkfd_process_info *process_info, 1325 struct amdgpu_sync *sync) 1326 { 1327 struct amdgpu_vm *peer_vm; 1328 int ret; 1329 1330 list_for_each_entry(peer_vm, &process_info->vm_list_head, 1331 vm_list_node) { 1332 ret = vm_update_pds(peer_vm, sync); 1333 if (ret) 1334 return ret; 1335 } 1336 1337 return 0; 1338 } 1339 1340 static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, 1341 struct dma_fence **ef) 1342 { 1343 struct amdkfd_process_info *info = NULL; 1344 int ret; 1345 1346 if (!*process_info) { 1347 info = kzalloc(sizeof(*info), GFP_KERNEL); 1348 if (!info) 1349 return -ENOMEM; 1350 1351 mutex_init(&info->lock); 1352 INIT_LIST_HEAD(&info->vm_list_head); 1353 INIT_LIST_HEAD(&info->kfd_bo_list); 1354 INIT_LIST_HEAD(&info->userptr_valid_list); 1355 INIT_LIST_HEAD(&info->userptr_inval_list); 1356 1357 info->eviction_fence = 1358 amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1), 1359 current->mm, 1360 NULL); 1361 if (!info->eviction_fence) { 1362 pr_err("Failed to create eviction fence\n"); 1363 ret = -ENOMEM; 1364 goto create_evict_fence_fail; 1365 } 1366 1367 info->pid = get_task_pid(current->group_leader, PIDTYPE_PID); 1368 atomic_set(&info->evicted_bos, 0); 1369 INIT_DELAYED_WORK(&info->restore_userptr_work, 1370 amdgpu_amdkfd_restore_userptr_worker); 1371 1372 *process_info = info; 1373 *ef = dma_fence_get(&info->eviction_fence->base); 1374 } 1375 1376 vm->process_info = *process_info; 1377 1378 /* Validate page directory and attach eviction fence */ 1379 ret = amdgpu_bo_reserve(vm->root.bo, true); 1380 if (ret) 1381 goto reserve_pd_fail; 1382 ret = vm_validate_pt_pd_bos(vm); 1383 if (ret) { 1384 pr_err("validate_pt_pd_bos() failed\n"); 1385 goto validate_pd_fail; 1386 } 1387 ret = amdgpu_bo_sync_wait(vm->root.bo, 1388 AMDGPU_FENCE_OWNER_KFD, false); 1389 if (ret) 1390 goto wait_pd_fail; 1391 ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1); 1392 if (ret) 1393 goto reserve_shared_fail; 1394 dma_resv_add_fence(vm->root.bo->tbo.base.resv, 1395 &vm->process_info->eviction_fence->base, 1396 DMA_RESV_USAGE_BOOKKEEP); 1397 amdgpu_bo_unreserve(vm->root.bo); 1398 1399 /* Update process info */ 1400 mutex_lock(&vm->process_info->lock); 1401 list_add_tail(&vm->vm_list_node, 1402 &(vm->process_info->vm_list_head)); 1403 vm->process_info->n_vms++; 1404 mutex_unlock(&vm->process_info->lock); 1405 1406 return 0; 1407 1408 reserve_shared_fail: 1409 wait_pd_fail: 1410 validate_pd_fail: 1411 amdgpu_bo_unreserve(vm->root.bo); 1412 reserve_pd_fail: 1413 vm->process_info = NULL; 1414 if (info) { 1415 /* Two fence references: one in info and one in *ef */ 1416 dma_fence_put(&info->eviction_fence->base); 1417 dma_fence_put(*ef); 1418 *ef = NULL; 1419 *process_info = NULL; 1420 put_pid(info->pid); 1421 create_evict_fence_fail: 1422 mutex_destroy(&info->lock); 1423 kfree(info); 1424 } 1425 return ret; 1426 } 1427 1428 /** 1429 * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria 1430 * @bo: Handle of buffer object being pinned 1431 * @domain: Domain into which BO should be pinned 1432 * 1433 * - USERPTR BOs are UNPINNABLE and will return error 1434 * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their 1435 * PIN count incremented. It is valid to PIN a BO multiple times 1436 * 1437 * Return: ZERO if successful in pinning, Non-Zero in case of error. 1438 */ 1439 static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain) 1440 { 1441 int ret = 0; 1442 1443 ret = amdgpu_bo_reserve(bo, false); 1444 if (unlikely(ret)) 1445 return ret; 1446 1447 ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0); 1448 if (ret) 1449 pr_err("Error in Pinning BO to domain: %d\n", domain); 1450 1451 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); 1452 amdgpu_bo_unreserve(bo); 1453 1454 return ret; 1455 } 1456 1457 /** 1458 * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria 1459 * @bo: Handle of buffer object being unpinned 1460 * 1461 * - Is a illegal request for USERPTR BOs and is ignored 1462 * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their 1463 * PIN count decremented. Calls to UNPIN must balance calls to PIN 1464 */ 1465 static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo) 1466 { 1467 int ret = 0; 1468 1469 ret = amdgpu_bo_reserve(bo, false); 1470 if (unlikely(ret)) 1471 return; 1472 1473 amdgpu_bo_unpin(bo); 1474 amdgpu_bo_unreserve(bo); 1475 } 1476 1477 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev, 1478 struct file *filp, u32 pasid, 1479 void **process_info, 1480 struct dma_fence **ef) 1481 { 1482 struct amdgpu_fpriv *drv_priv; 1483 struct amdgpu_vm *avm; 1484 int ret; 1485 1486 ret = amdgpu_file_to_fpriv(filp, &drv_priv); 1487 if (ret) 1488 return ret; 1489 avm = &drv_priv->vm; 1490 1491 /* Already a compute VM? */ 1492 if (avm->process_info) 1493 return -EINVAL; 1494 1495 /* Free the original amdgpu allocated pasid, 1496 * will be replaced with kfd allocated pasid. 1497 */ 1498 if (avm->pasid) { 1499 amdgpu_pasid_free(avm->pasid); 1500 amdgpu_vm_set_pasid(adev, avm, 0); 1501 } 1502 1503 /* Convert VM into a compute VM */ 1504 ret = amdgpu_vm_make_compute(adev, avm); 1505 if (ret) 1506 return ret; 1507 1508 ret = amdgpu_vm_set_pasid(adev, avm, pasid); 1509 if (ret) 1510 return ret; 1511 /* Initialize KFD part of the VM and process info */ 1512 ret = init_kfd_vm(avm, process_info, ef); 1513 if (ret) 1514 return ret; 1515 1516 amdgpu_vm_set_task_info(avm); 1517 1518 return 0; 1519 } 1520 1521 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, 1522 struct amdgpu_vm *vm) 1523 { 1524 struct amdkfd_process_info *process_info = vm->process_info; 1525 1526 if (!process_info) 1527 return; 1528 1529 /* Update process info */ 1530 mutex_lock(&process_info->lock); 1531 process_info->n_vms--; 1532 list_del(&vm->vm_list_node); 1533 mutex_unlock(&process_info->lock); 1534 1535 vm->process_info = NULL; 1536 1537 /* Release per-process resources when last compute VM is destroyed */ 1538 if (!process_info->n_vms) { 1539 WARN_ON(!list_empty(&process_info->kfd_bo_list)); 1540 WARN_ON(!list_empty(&process_info->userptr_valid_list)); 1541 WARN_ON(!list_empty(&process_info->userptr_inval_list)); 1542 1543 dma_fence_put(&process_info->eviction_fence->base); 1544 cancel_delayed_work_sync(&process_info->restore_userptr_work); 1545 put_pid(process_info->pid); 1546 mutex_destroy(&process_info->lock); 1547 kfree(process_info); 1548 } 1549 } 1550 1551 void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev, 1552 void *drm_priv) 1553 { 1554 struct amdgpu_vm *avm; 1555 1556 if (WARN_ON(!adev || !drm_priv)) 1557 return; 1558 1559 avm = drm_priv_to_vm(drm_priv); 1560 1561 pr_debug("Releasing process vm %p\n", avm); 1562 1563 /* The original pasid of amdgpu vm has already been 1564 * released during making a amdgpu vm to a compute vm 1565 * The current pasid is managed by kfd and will be 1566 * released on kfd process destroy. Set amdgpu pasid 1567 * to 0 to avoid duplicate release. 1568 */ 1569 amdgpu_vm_release_compute(adev, avm); 1570 } 1571 1572 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv) 1573 { 1574 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 1575 struct amdgpu_bo *pd = avm->root.bo; 1576 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); 1577 1578 if (adev->asic_type < CHIP_VEGA10) 1579 return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT; 1580 return avm->pd_phys_addr; 1581 } 1582 1583 void amdgpu_amdkfd_block_mmu_notifications(void *p) 1584 { 1585 struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p; 1586 1587 mutex_lock(&pinfo->lock); 1588 WRITE_ONCE(pinfo->block_mmu_notifications, true); 1589 mutex_unlock(&pinfo->lock); 1590 } 1591 1592 int amdgpu_amdkfd_criu_resume(void *p) 1593 { 1594 int ret = 0; 1595 struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p; 1596 1597 mutex_lock(&pinfo->lock); 1598 pr_debug("scheduling work\n"); 1599 atomic_inc(&pinfo->evicted_bos); 1600 if (!READ_ONCE(pinfo->block_mmu_notifications)) { 1601 ret = -EINVAL; 1602 goto out_unlock; 1603 } 1604 WRITE_ONCE(pinfo->block_mmu_notifications, false); 1605 schedule_delayed_work(&pinfo->restore_userptr_work, 0); 1606 1607 out_unlock: 1608 mutex_unlock(&pinfo->lock); 1609 return ret; 1610 } 1611 1612 size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev) 1613 { 1614 uint64_t reserved_for_pt = 1615 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); 1616 size_t available; 1617 1618 spin_lock(&kfd_mem_limit.mem_limit_lock); 1619 available = adev->gmc.real_vram_size 1620 - adev->kfd.vram_used_aligned 1621 - atomic64_read(&adev->vram_pin_size) 1622 - reserved_for_pt; 1623 spin_unlock(&kfd_mem_limit.mem_limit_lock); 1624 1625 return ALIGN_DOWN(available, VRAM_AVAILABLITY_ALIGN); 1626 } 1627 1628 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( 1629 struct amdgpu_device *adev, uint64_t va, uint64_t size, 1630 void *drm_priv, struct kgd_mem **mem, 1631 uint64_t *offset, uint32_t flags, bool criu_resume) 1632 { 1633 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 1634 enum ttm_bo_type bo_type = ttm_bo_type_device; 1635 struct sg_table *sg = NULL; 1636 uint64_t user_addr = 0; 1637 struct amdgpu_bo *bo; 1638 struct drm_gem_object *gobj = NULL; 1639 u32 domain, alloc_domain; 1640 u64 alloc_flags; 1641 int ret; 1642 1643 /* 1644 * Check on which domain to allocate BO 1645 */ 1646 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 1647 domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM; 1648 alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE; 1649 alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ? 1650 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0; 1651 } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { 1652 domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT; 1653 alloc_flags = 0; 1654 } else { 1655 domain = AMDGPU_GEM_DOMAIN_GTT; 1656 alloc_domain = AMDGPU_GEM_DOMAIN_CPU; 1657 alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE; 1658 1659 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 1660 if (!offset || !*offset) 1661 return -EINVAL; 1662 user_addr = untagged_addr(*offset); 1663 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 1664 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 1665 bo_type = ttm_bo_type_sg; 1666 if (size > UINT_MAX) 1667 return -EINVAL; 1668 sg = create_sg_table(*offset, size); 1669 if (!sg) 1670 return -ENOMEM; 1671 } else { 1672 return -EINVAL; 1673 } 1674 } 1675 1676 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); 1677 if (!*mem) { 1678 ret = -ENOMEM; 1679 goto err; 1680 } 1681 INIT_LIST_HEAD(&(*mem)->attachments); 1682 mutex_init(&(*mem)->lock); 1683 (*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM); 1684 1685 /* Workaround for AQL queue wraparound bug. Map the same 1686 * memory twice. That means we only actually allocate half 1687 * the memory. 1688 */ 1689 if ((*mem)->aql_queue) 1690 size = size >> 1; 1691 1692 (*mem)->alloc_flags = flags; 1693 1694 amdgpu_sync_create(&(*mem)->sync); 1695 1696 ret = amdgpu_amdkfd_reserve_mem_limit(adev, size, flags); 1697 if (ret) { 1698 pr_debug("Insufficient memory\n"); 1699 goto err_reserve_limit; 1700 } 1701 1702 pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s\n", 1703 va, size, domain_string(alloc_domain)); 1704 1705 ret = amdgpu_gem_object_create(adev, size, 1, alloc_domain, alloc_flags, 1706 bo_type, NULL, &gobj); 1707 if (ret) { 1708 pr_debug("Failed to create BO on domain %s. ret %d\n", 1709 domain_string(alloc_domain), ret); 1710 goto err_bo_create; 1711 } 1712 ret = drm_vma_node_allow(&gobj->vma_node, drm_priv); 1713 if (ret) { 1714 pr_debug("Failed to allow vma node access. ret %d\n", ret); 1715 goto err_node_allow; 1716 } 1717 bo = gem_to_amdgpu_bo(gobj); 1718 if (bo_type == ttm_bo_type_sg) { 1719 bo->tbo.sg = sg; 1720 bo->tbo.ttm->sg = sg; 1721 } 1722 bo->kfd_bo = *mem; 1723 (*mem)->bo = bo; 1724 if (user_addr) 1725 bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO; 1726 1727 (*mem)->va = va; 1728 (*mem)->domain = domain; 1729 (*mem)->mapped_to_gpu_memory = 0; 1730 (*mem)->process_info = avm->process_info; 1731 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr); 1732 1733 if (user_addr) { 1734 pr_debug("creating userptr BO for user_addr = %llx\n", user_addr); 1735 ret = init_user_pages(*mem, user_addr, criu_resume); 1736 if (ret) 1737 goto allocate_init_user_pages_failed; 1738 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 1739 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 1740 ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT); 1741 if (ret) { 1742 pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n"); 1743 goto err_pin_bo; 1744 } 1745 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; 1746 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; 1747 } 1748 1749 if (offset) 1750 *offset = amdgpu_bo_mmap_offset(bo); 1751 1752 return 0; 1753 1754 allocate_init_user_pages_failed: 1755 err_pin_bo: 1756 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info); 1757 drm_vma_node_revoke(&gobj->vma_node, drm_priv); 1758 err_node_allow: 1759 /* Don't unreserve system mem limit twice */ 1760 goto err_reserve_limit; 1761 err_bo_create: 1762 amdgpu_amdkfd_unreserve_mem_limit(adev, size, flags); 1763 err_reserve_limit: 1764 mutex_destroy(&(*mem)->lock); 1765 if (gobj) 1766 drm_gem_object_put(gobj); 1767 else 1768 kfree(*mem); 1769 err: 1770 if (sg) { 1771 sg_free_table(sg); 1772 kfree(sg); 1773 } 1774 return ret; 1775 } 1776 1777 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( 1778 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv, 1779 uint64_t *size) 1780 { 1781 struct amdkfd_process_info *process_info = mem->process_info; 1782 unsigned long bo_size = mem->bo->tbo.base.size; 1783 bool use_release_notifier = (mem->bo->kfd_bo == mem); 1784 struct kfd_mem_attachment *entry, *tmp; 1785 struct bo_vm_reservation_context ctx; 1786 struct ttm_validate_buffer *bo_list_entry; 1787 unsigned int mapped_to_gpu_memory; 1788 int ret; 1789 bool is_imported = false; 1790 1791 mutex_lock(&mem->lock); 1792 1793 /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */ 1794 if (mem->alloc_flags & 1795 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 1796 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 1797 amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo); 1798 } 1799 1800 mapped_to_gpu_memory = mem->mapped_to_gpu_memory; 1801 is_imported = mem->is_imported; 1802 mutex_unlock(&mem->lock); 1803 /* lock is not needed after this, since mem is unused and will 1804 * be freed anyway 1805 */ 1806 1807 if (mapped_to_gpu_memory > 0) { 1808 pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n", 1809 mem->va, bo_size); 1810 return -EBUSY; 1811 } 1812 1813 /* Make sure restore workers don't access the BO any more */ 1814 bo_list_entry = &mem->validate_list; 1815 mutex_lock(&process_info->lock); 1816 list_del(&bo_list_entry->head); 1817 mutex_unlock(&process_info->lock); 1818 1819 /* No more MMU notifiers */ 1820 amdgpu_mn_unregister(mem->bo); 1821 1822 ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx); 1823 if (unlikely(ret)) 1824 return ret; 1825 1826 /* The eviction fence should be removed by the last unmap. 1827 * TODO: Log an error condition if the bo still has the eviction fence 1828 * attached 1829 */ 1830 amdgpu_amdkfd_remove_eviction_fence(mem->bo, 1831 process_info->eviction_fence); 1832 pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va, 1833 mem->va + bo_size * (1 + mem->aql_queue)); 1834 1835 /* Remove from VM internal data structures */ 1836 list_for_each_entry_safe(entry, tmp, &mem->attachments, list) 1837 kfd_mem_detach(entry); 1838 1839 ret = unreserve_bo_and_vms(&ctx, false, false); 1840 1841 /* Free the sync object */ 1842 amdgpu_sync_free(&mem->sync); 1843 1844 /* If the SG is not NULL, it's one we created for a doorbell or mmio 1845 * remap BO. We need to free it. 1846 */ 1847 if (mem->bo->tbo.sg) { 1848 sg_free_table(mem->bo->tbo.sg); 1849 kfree(mem->bo->tbo.sg); 1850 } 1851 1852 /* Update the size of the BO being freed if it was allocated from 1853 * VRAM and is not imported. 1854 */ 1855 if (size) { 1856 if ((mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM) && 1857 (!is_imported)) 1858 *size = bo_size; 1859 else 1860 *size = 0; 1861 } 1862 1863 /* Free the BO*/ 1864 drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv); 1865 if (mem->dmabuf) 1866 dma_buf_put(mem->dmabuf); 1867 mutex_destroy(&mem->lock); 1868 1869 /* If this releases the last reference, it will end up calling 1870 * amdgpu_amdkfd_release_notify and kfree the mem struct. That's why 1871 * this needs to be the last call here. 1872 */ 1873 drm_gem_object_put(&mem->bo->tbo.base); 1874 1875 /* 1876 * For kgd_mem allocated in amdgpu_amdkfd_gpuvm_import_dmabuf(), 1877 * explicitly free it here. 1878 */ 1879 if (!use_release_notifier) 1880 kfree(mem); 1881 1882 return ret; 1883 } 1884 1885 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( 1886 struct amdgpu_device *adev, struct kgd_mem *mem, 1887 void *drm_priv) 1888 { 1889 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 1890 int ret; 1891 struct amdgpu_bo *bo; 1892 uint32_t domain; 1893 struct kfd_mem_attachment *entry; 1894 struct bo_vm_reservation_context ctx; 1895 unsigned long bo_size; 1896 bool is_invalid_userptr = false; 1897 1898 bo = mem->bo; 1899 if (!bo) { 1900 pr_err("Invalid BO when mapping memory to GPU\n"); 1901 return -EINVAL; 1902 } 1903 1904 /* Make sure restore is not running concurrently. Since we 1905 * don't map invalid userptr BOs, we rely on the next restore 1906 * worker to do the mapping 1907 */ 1908 mutex_lock(&mem->process_info->lock); 1909 1910 /* Lock mmap-sem. If we find an invalid userptr BO, we can be 1911 * sure that the MMU notifier is no longer running 1912 * concurrently and the queues are actually stopped 1913 */ 1914 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { 1915 mmap_write_lock(current->mm); 1916 is_invalid_userptr = atomic_read(&mem->invalid); 1917 mmap_write_unlock(current->mm); 1918 } 1919 1920 mutex_lock(&mem->lock); 1921 1922 domain = mem->domain; 1923 bo_size = bo->tbo.base.size; 1924 1925 pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n", 1926 mem->va, 1927 mem->va + bo_size * (1 + mem->aql_queue), 1928 avm, domain_string(domain)); 1929 1930 if (!kfd_mem_is_attached(avm, mem)) { 1931 ret = kfd_mem_attach(adev, mem, avm, mem->aql_queue); 1932 if (ret) 1933 goto out; 1934 } 1935 1936 ret = reserve_bo_and_vm(mem, avm, &ctx); 1937 if (unlikely(ret)) 1938 goto out; 1939 1940 /* Userptr can be marked as "not invalid", but not actually be 1941 * validated yet (still in the system domain). In that case 1942 * the queues are still stopped and we can leave mapping for 1943 * the next restore worker 1944 */ 1945 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && 1946 bo->tbo.resource->mem_type == TTM_PL_SYSTEM) 1947 is_invalid_userptr = true; 1948 1949 ret = vm_validate_pt_pd_bos(avm); 1950 if (unlikely(ret)) 1951 goto out_unreserve; 1952 1953 if (mem->mapped_to_gpu_memory == 0 && 1954 !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { 1955 /* Validate BO only once. The eviction fence gets added to BO 1956 * the first time it is mapped. Validate will wait for all 1957 * background evictions to complete. 1958 */ 1959 ret = amdgpu_amdkfd_bo_validate(bo, domain, true); 1960 if (ret) { 1961 pr_debug("Validate failed\n"); 1962 goto out_unreserve; 1963 } 1964 } 1965 1966 list_for_each_entry(entry, &mem->attachments, list) { 1967 if (entry->bo_va->base.vm != avm || entry->is_mapped) 1968 continue; 1969 1970 pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n", 1971 entry->va, entry->va + bo_size, entry); 1972 1973 ret = map_bo_to_gpuvm(mem, entry, ctx.sync, 1974 is_invalid_userptr); 1975 if (ret) { 1976 pr_err("Failed to map bo to gpuvm\n"); 1977 goto out_unreserve; 1978 } 1979 1980 ret = vm_update_pds(avm, ctx.sync); 1981 if (ret) { 1982 pr_err("Failed to update page directories\n"); 1983 goto out_unreserve; 1984 } 1985 1986 entry->is_mapped = true; 1987 mem->mapped_to_gpu_memory++; 1988 pr_debug("\t INC mapping count %d\n", 1989 mem->mapped_to_gpu_memory); 1990 } 1991 1992 if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->tbo.pin_count) 1993 dma_resv_add_fence(bo->tbo.base.resv, 1994 &avm->process_info->eviction_fence->base, 1995 DMA_RESV_USAGE_BOOKKEEP); 1996 ret = unreserve_bo_and_vms(&ctx, false, false); 1997 1998 goto out; 1999 2000 out_unreserve: 2001 unreserve_bo_and_vms(&ctx, false, false); 2002 out: 2003 mutex_unlock(&mem->process_info->lock); 2004 mutex_unlock(&mem->lock); 2005 return ret; 2006 } 2007 2008 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( 2009 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv) 2010 { 2011 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 2012 struct amdkfd_process_info *process_info = avm->process_info; 2013 unsigned long bo_size = mem->bo->tbo.base.size; 2014 struct kfd_mem_attachment *entry; 2015 struct bo_vm_reservation_context ctx; 2016 int ret; 2017 2018 mutex_lock(&mem->lock); 2019 2020 ret = reserve_bo_and_cond_vms(mem, avm, BO_VM_MAPPED, &ctx); 2021 if (unlikely(ret)) 2022 goto out; 2023 /* If no VMs were reserved, it means the BO wasn't actually mapped */ 2024 if (ctx.n_vms == 0) { 2025 ret = -EINVAL; 2026 goto unreserve_out; 2027 } 2028 2029 ret = vm_validate_pt_pd_bos(avm); 2030 if (unlikely(ret)) 2031 goto unreserve_out; 2032 2033 pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n", 2034 mem->va, 2035 mem->va + bo_size * (1 + mem->aql_queue), 2036 avm); 2037 2038 list_for_each_entry(entry, &mem->attachments, list) { 2039 if (entry->bo_va->base.vm != avm || !entry->is_mapped) 2040 continue; 2041 2042 pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n", 2043 entry->va, entry->va + bo_size, entry); 2044 2045 unmap_bo_from_gpuvm(mem, entry, ctx.sync); 2046 entry->is_mapped = false; 2047 2048 mem->mapped_to_gpu_memory--; 2049 pr_debug("\t DEC mapping count %d\n", 2050 mem->mapped_to_gpu_memory); 2051 } 2052 2053 /* If BO is unmapped from all VMs, unfence it. It can be evicted if 2054 * required. 2055 */ 2056 if (mem->mapped_to_gpu_memory == 0 && 2057 !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && 2058 !mem->bo->tbo.pin_count) 2059 amdgpu_amdkfd_remove_eviction_fence(mem->bo, 2060 process_info->eviction_fence); 2061 2062 unreserve_out: 2063 unreserve_bo_and_vms(&ctx, false, false); 2064 out: 2065 mutex_unlock(&mem->lock); 2066 return ret; 2067 } 2068 2069 int amdgpu_amdkfd_gpuvm_sync_memory( 2070 struct amdgpu_device *adev, struct kgd_mem *mem, bool intr) 2071 { 2072 struct amdgpu_sync sync; 2073 int ret; 2074 2075 amdgpu_sync_create(&sync); 2076 2077 mutex_lock(&mem->lock); 2078 amdgpu_sync_clone(&mem->sync, &sync); 2079 mutex_unlock(&mem->lock); 2080 2081 ret = amdgpu_sync_wait(&sync, intr); 2082 amdgpu_sync_free(&sync); 2083 return ret; 2084 } 2085 2086 /** 2087 * amdgpu_amdkfd_map_gtt_bo_to_gart - Map BO to GART and increment reference count 2088 * @adev: Device to which allocated BO belongs 2089 * @bo: Buffer object to be mapped 2090 * 2091 * Before return, bo reference count is incremented. To release the reference and unpin/ 2092 * unmap the BO, call amdgpu_amdkfd_free_gtt_mem. 2093 */ 2094 int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_device *adev, struct amdgpu_bo *bo) 2095 { 2096 int ret; 2097 2098 ret = amdgpu_bo_reserve(bo, true); 2099 if (ret) { 2100 pr_err("Failed to reserve bo. ret %d\n", ret); 2101 goto err_reserve_bo_failed; 2102 } 2103 2104 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); 2105 if (ret) { 2106 pr_err("Failed to pin bo. ret %d\n", ret); 2107 goto err_pin_bo_failed; 2108 } 2109 2110 ret = amdgpu_ttm_alloc_gart(&bo->tbo); 2111 if (ret) { 2112 pr_err("Failed to bind bo to GART. ret %d\n", ret); 2113 goto err_map_bo_gart_failed; 2114 } 2115 2116 amdgpu_amdkfd_remove_eviction_fence( 2117 bo, bo->kfd_bo->process_info->eviction_fence); 2118 2119 amdgpu_bo_unreserve(bo); 2120 2121 bo = amdgpu_bo_ref(bo); 2122 2123 return 0; 2124 2125 err_map_bo_gart_failed: 2126 amdgpu_bo_unpin(bo); 2127 err_pin_bo_failed: 2128 amdgpu_bo_unreserve(bo); 2129 err_reserve_bo_failed: 2130 2131 return ret; 2132 } 2133 2134 /** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Map a GTT BO for kernel CPU access 2135 * 2136 * @mem: Buffer object to be mapped for CPU access 2137 * @kptr[out]: pointer in kernel CPU address space 2138 * @size[out]: size of the buffer 2139 * 2140 * Pins the BO and maps it for kernel CPU access. The eviction fence is removed 2141 * from the BO, since pinned BOs cannot be evicted. The bo must remain on the 2142 * validate_list, so the GPU mapping can be restored after a page table was 2143 * evicted. 2144 * 2145 * Return: 0 on success, error code on failure 2146 */ 2147 int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem, 2148 void **kptr, uint64_t *size) 2149 { 2150 int ret; 2151 struct amdgpu_bo *bo = mem->bo; 2152 2153 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { 2154 pr_err("userptr can't be mapped to kernel\n"); 2155 return -EINVAL; 2156 } 2157 2158 mutex_lock(&mem->process_info->lock); 2159 2160 ret = amdgpu_bo_reserve(bo, true); 2161 if (ret) { 2162 pr_err("Failed to reserve bo. ret %d\n", ret); 2163 goto bo_reserve_failed; 2164 } 2165 2166 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); 2167 if (ret) { 2168 pr_err("Failed to pin bo. ret %d\n", ret); 2169 goto pin_failed; 2170 } 2171 2172 ret = amdgpu_bo_kmap(bo, kptr); 2173 if (ret) { 2174 pr_err("Failed to map bo to kernel. ret %d\n", ret); 2175 goto kmap_failed; 2176 } 2177 2178 amdgpu_amdkfd_remove_eviction_fence( 2179 bo, mem->process_info->eviction_fence); 2180 2181 if (size) 2182 *size = amdgpu_bo_size(bo); 2183 2184 amdgpu_bo_unreserve(bo); 2185 2186 mutex_unlock(&mem->process_info->lock); 2187 return 0; 2188 2189 kmap_failed: 2190 amdgpu_bo_unpin(bo); 2191 pin_failed: 2192 amdgpu_bo_unreserve(bo); 2193 bo_reserve_failed: 2194 mutex_unlock(&mem->process_info->lock); 2195 2196 return ret; 2197 } 2198 2199 /** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Unmap a GTT BO for kernel CPU access 2200 * 2201 * @mem: Buffer object to be unmapped for CPU access 2202 * 2203 * Removes the kernel CPU mapping and unpins the BO. It does not restore the 2204 * eviction fence, so this function should only be used for cleanup before the 2205 * BO is destroyed. 2206 */ 2207 void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem) 2208 { 2209 struct amdgpu_bo *bo = mem->bo; 2210 2211 amdgpu_bo_reserve(bo, true); 2212 amdgpu_bo_kunmap(bo); 2213 amdgpu_bo_unpin(bo); 2214 amdgpu_bo_unreserve(bo); 2215 } 2216 2217 int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev, 2218 struct kfd_vm_fault_info *mem) 2219 { 2220 if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) { 2221 *mem = *adev->gmc.vm_fault_info; 2222 mb(); /* make sure read happened */ 2223 atomic_set(&adev->gmc.vm_fault_info_updated, 0); 2224 } 2225 return 0; 2226 } 2227 2228 int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev, 2229 struct dma_buf *dma_buf, 2230 uint64_t va, void *drm_priv, 2231 struct kgd_mem **mem, uint64_t *size, 2232 uint64_t *mmap_offset) 2233 { 2234 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 2235 struct drm_gem_object *obj; 2236 struct amdgpu_bo *bo; 2237 int ret; 2238 2239 if (dma_buf->ops != &amdgpu_dmabuf_ops) 2240 /* Can't handle non-graphics buffers */ 2241 return -EINVAL; 2242 2243 obj = dma_buf->priv; 2244 if (drm_to_adev(obj->dev) != adev) 2245 /* Can't handle buffers from other devices */ 2246 return -EINVAL; 2247 2248 bo = gem_to_amdgpu_bo(obj); 2249 if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | 2250 AMDGPU_GEM_DOMAIN_GTT))) 2251 /* Only VRAM and GTT BOs are supported */ 2252 return -EINVAL; 2253 2254 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); 2255 if (!*mem) 2256 return -ENOMEM; 2257 2258 ret = drm_vma_node_allow(&obj->vma_node, drm_priv); 2259 if (ret) { 2260 kfree(mem); 2261 return ret; 2262 } 2263 2264 if (size) 2265 *size = amdgpu_bo_size(bo); 2266 2267 if (mmap_offset) 2268 *mmap_offset = amdgpu_bo_mmap_offset(bo); 2269 2270 INIT_LIST_HEAD(&(*mem)->attachments); 2271 mutex_init(&(*mem)->lock); 2272 2273 (*mem)->alloc_flags = 2274 ((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? 2275 KFD_IOC_ALLOC_MEM_FLAGS_VRAM : KFD_IOC_ALLOC_MEM_FLAGS_GTT) 2276 | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE 2277 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; 2278 2279 drm_gem_object_get(&bo->tbo.base); 2280 (*mem)->bo = bo; 2281 (*mem)->va = va; 2282 (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? 2283 AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT; 2284 (*mem)->mapped_to_gpu_memory = 0; 2285 (*mem)->process_info = avm->process_info; 2286 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false); 2287 amdgpu_sync_create(&(*mem)->sync); 2288 (*mem)->is_imported = true; 2289 2290 return 0; 2291 } 2292 2293 /* Evict a userptr BO by stopping the queues if necessary 2294 * 2295 * Runs in MMU notifier, may be in RECLAIM_FS context. This means it 2296 * cannot do any memory allocations, and cannot take any locks that 2297 * are held elsewhere while allocating memory. Therefore this is as 2298 * simple as possible, using atomic counters. 2299 * 2300 * It doesn't do anything to the BO itself. The real work happens in 2301 * restore, where we get updated page addresses. This function only 2302 * ensures that GPU access to the BO is stopped. 2303 */ 2304 int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, 2305 struct mm_struct *mm) 2306 { 2307 struct amdkfd_process_info *process_info = mem->process_info; 2308 int evicted_bos; 2309 int r = 0; 2310 2311 /* Do not process MMU notifications until stage-4 IOCTL is received */ 2312 if (READ_ONCE(process_info->block_mmu_notifications)) 2313 return 0; 2314 2315 atomic_inc(&mem->invalid); 2316 evicted_bos = atomic_inc_return(&process_info->evicted_bos); 2317 if (evicted_bos == 1) { 2318 /* First eviction, stop the queues */ 2319 r = kgd2kfd_quiesce_mm(mm, KFD_QUEUE_EVICTION_TRIGGER_USERPTR); 2320 if (r) 2321 pr_err("Failed to quiesce KFD\n"); 2322 schedule_delayed_work(&process_info->restore_userptr_work, 2323 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); 2324 } 2325 2326 return r; 2327 } 2328 2329 /* Update invalid userptr BOs 2330 * 2331 * Moves invalidated (evicted) userptr BOs from userptr_valid_list to 2332 * userptr_inval_list and updates user pages for all BOs that have 2333 * been invalidated since their last update. 2334 */ 2335 static int update_invalid_user_pages(struct amdkfd_process_info *process_info, 2336 struct mm_struct *mm) 2337 { 2338 struct kgd_mem *mem, *tmp_mem; 2339 struct amdgpu_bo *bo; 2340 struct ttm_operation_ctx ctx = { false, false }; 2341 int invalid, ret; 2342 2343 /* Move all invalidated BOs to the userptr_inval_list and 2344 * release their user pages by migration to the CPU domain 2345 */ 2346 list_for_each_entry_safe(mem, tmp_mem, 2347 &process_info->userptr_valid_list, 2348 validate_list.head) { 2349 if (!atomic_read(&mem->invalid)) 2350 continue; /* BO is still valid */ 2351 2352 bo = mem->bo; 2353 2354 if (amdgpu_bo_reserve(bo, true)) 2355 return -EAGAIN; 2356 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 2357 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 2358 amdgpu_bo_unreserve(bo); 2359 if (ret) { 2360 pr_err("%s: Failed to invalidate userptr BO\n", 2361 __func__); 2362 return -EAGAIN; 2363 } 2364 2365 list_move_tail(&mem->validate_list.head, 2366 &process_info->userptr_inval_list); 2367 } 2368 2369 if (list_empty(&process_info->userptr_inval_list)) 2370 return 0; /* All evicted userptr BOs were freed */ 2371 2372 /* Go through userptr_inval_list and update any invalid user_pages */ 2373 list_for_each_entry(mem, &process_info->userptr_inval_list, 2374 validate_list.head) { 2375 invalid = atomic_read(&mem->invalid); 2376 if (!invalid) 2377 /* BO hasn't been invalidated since the last 2378 * revalidation attempt. Keep its BO list. 2379 */ 2380 continue; 2381 2382 bo = mem->bo; 2383 2384 /* Get updated user pages */ 2385 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages); 2386 if (ret) { 2387 pr_debug("Failed %d to get user pages\n", ret); 2388 2389 /* Return -EFAULT bad address error as success. It will 2390 * fail later with a VM fault if the GPU tries to access 2391 * it. Better than hanging indefinitely with stalled 2392 * user mode queues. 2393 * 2394 * Return other error -EBUSY or -ENOMEM to retry restore 2395 */ 2396 if (ret != -EFAULT) 2397 return ret; 2398 } else { 2399 2400 /* 2401 * FIXME: Cannot ignore the return code, must hold 2402 * notifier_lock 2403 */ 2404 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); 2405 } 2406 2407 /* Mark the BO as valid unless it was invalidated 2408 * again concurrently. 2409 */ 2410 if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid) 2411 return -EAGAIN; 2412 } 2413 2414 return 0; 2415 } 2416 2417 /* Validate invalid userptr BOs 2418 * 2419 * Validates BOs on the userptr_inval_list, and moves them back to the 2420 * userptr_valid_list. Also updates GPUVM page tables with new page 2421 * addresses and waits for the page table updates to complete. 2422 */ 2423 static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) 2424 { 2425 struct amdgpu_bo_list_entry *pd_bo_list_entries; 2426 struct list_head resv_list, duplicates; 2427 struct ww_acquire_ctx ticket; 2428 struct amdgpu_sync sync; 2429 2430 struct amdgpu_vm *peer_vm; 2431 struct kgd_mem *mem, *tmp_mem; 2432 struct amdgpu_bo *bo; 2433 struct ttm_operation_ctx ctx = { false, false }; 2434 int i, ret; 2435 2436 pd_bo_list_entries = kcalloc(process_info->n_vms, 2437 sizeof(struct amdgpu_bo_list_entry), 2438 GFP_KERNEL); 2439 if (!pd_bo_list_entries) { 2440 pr_err("%s: Failed to allocate PD BO list entries\n", __func__); 2441 ret = -ENOMEM; 2442 goto out_no_mem; 2443 } 2444 2445 INIT_LIST_HEAD(&resv_list); 2446 INIT_LIST_HEAD(&duplicates); 2447 2448 /* Get all the page directory BOs that need to be reserved */ 2449 i = 0; 2450 list_for_each_entry(peer_vm, &process_info->vm_list_head, 2451 vm_list_node) 2452 amdgpu_vm_get_pd_bo(peer_vm, &resv_list, 2453 &pd_bo_list_entries[i++]); 2454 /* Add the userptr_inval_list entries to resv_list */ 2455 list_for_each_entry(mem, &process_info->userptr_inval_list, 2456 validate_list.head) { 2457 list_add_tail(&mem->resv_list.head, &resv_list); 2458 mem->resv_list.bo = mem->validate_list.bo; 2459 mem->resv_list.num_shared = mem->validate_list.num_shared; 2460 } 2461 2462 /* Reserve all BOs and page tables for validation */ 2463 ret = ttm_eu_reserve_buffers(&ticket, &resv_list, false, &duplicates); 2464 WARN(!list_empty(&duplicates), "Duplicates should be empty"); 2465 if (ret) 2466 goto out_free; 2467 2468 amdgpu_sync_create(&sync); 2469 2470 ret = process_validate_vms(process_info); 2471 if (ret) 2472 goto unreserve_out; 2473 2474 /* Validate BOs and update GPUVM page tables */ 2475 list_for_each_entry_safe(mem, tmp_mem, 2476 &process_info->userptr_inval_list, 2477 validate_list.head) { 2478 struct kfd_mem_attachment *attachment; 2479 2480 bo = mem->bo; 2481 2482 /* Validate the BO if we got user pages */ 2483 if (bo->tbo.ttm->pages[0]) { 2484 amdgpu_bo_placement_from_domain(bo, mem->domain); 2485 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 2486 if (ret) { 2487 pr_err("%s: failed to validate BO\n", __func__); 2488 goto unreserve_out; 2489 } 2490 } 2491 2492 list_move_tail(&mem->validate_list.head, 2493 &process_info->userptr_valid_list); 2494 2495 /* Update mapping. If the BO was not validated 2496 * (because we couldn't get user pages), this will 2497 * clear the page table entries, which will result in 2498 * VM faults if the GPU tries to access the invalid 2499 * memory. 2500 */ 2501 list_for_each_entry(attachment, &mem->attachments, list) { 2502 if (!attachment->is_mapped) 2503 continue; 2504 2505 kfd_mem_dmaunmap_attachment(mem, attachment); 2506 ret = update_gpuvm_pte(mem, attachment, &sync); 2507 if (ret) { 2508 pr_err("%s: update PTE failed\n", __func__); 2509 /* make sure this gets validated again */ 2510 atomic_inc(&mem->invalid); 2511 goto unreserve_out; 2512 } 2513 } 2514 } 2515 2516 /* Update page directories */ 2517 ret = process_update_pds(process_info, &sync); 2518 2519 unreserve_out: 2520 ttm_eu_backoff_reservation(&ticket, &resv_list); 2521 amdgpu_sync_wait(&sync, false); 2522 amdgpu_sync_free(&sync); 2523 out_free: 2524 kfree(pd_bo_list_entries); 2525 out_no_mem: 2526 2527 return ret; 2528 } 2529 2530 /* Worker callback to restore evicted userptr BOs 2531 * 2532 * Tries to update and validate all userptr BOs. If successful and no 2533 * concurrent evictions happened, the queues are restarted. Otherwise, 2534 * reschedule for another attempt later. 2535 */ 2536 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work) 2537 { 2538 struct delayed_work *dwork = to_delayed_work(work); 2539 struct amdkfd_process_info *process_info = 2540 container_of(dwork, struct amdkfd_process_info, 2541 restore_userptr_work); 2542 struct task_struct *usertask; 2543 struct mm_struct *mm; 2544 int evicted_bos; 2545 2546 evicted_bos = atomic_read(&process_info->evicted_bos); 2547 if (!evicted_bos) 2548 return; 2549 2550 /* Reference task and mm in case of concurrent process termination */ 2551 usertask = get_pid_task(process_info->pid, PIDTYPE_PID); 2552 if (!usertask) 2553 return; 2554 mm = get_task_mm(usertask); 2555 if (!mm) { 2556 put_task_struct(usertask); 2557 return; 2558 } 2559 2560 mutex_lock(&process_info->lock); 2561 2562 if (update_invalid_user_pages(process_info, mm)) 2563 goto unlock_out; 2564 /* userptr_inval_list can be empty if all evicted userptr BOs 2565 * have been freed. In that case there is nothing to validate 2566 * and we can just restart the queues. 2567 */ 2568 if (!list_empty(&process_info->userptr_inval_list)) { 2569 if (atomic_read(&process_info->evicted_bos) != evicted_bos) 2570 goto unlock_out; /* Concurrent eviction, try again */ 2571 2572 if (validate_invalid_user_pages(process_info)) 2573 goto unlock_out; 2574 } 2575 /* Final check for concurrent evicton and atomic update. If 2576 * another eviction happens after successful update, it will 2577 * be a first eviction that calls quiesce_mm. The eviction 2578 * reference counting inside KFD will handle this case. 2579 */ 2580 if (atomic_cmpxchg(&process_info->evicted_bos, evicted_bos, 0) != 2581 evicted_bos) 2582 goto unlock_out; 2583 evicted_bos = 0; 2584 if (kgd2kfd_resume_mm(mm)) { 2585 pr_err("%s: Failed to resume KFD\n", __func__); 2586 /* No recovery from this failure. Probably the CP is 2587 * hanging. No point trying again. 2588 */ 2589 } 2590 2591 unlock_out: 2592 mutex_unlock(&process_info->lock); 2593 2594 /* If validation failed, reschedule another attempt */ 2595 if (evicted_bos) { 2596 schedule_delayed_work(&process_info->restore_userptr_work, 2597 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); 2598 2599 kfd_smi_event_queue_restore_rescheduled(mm); 2600 } 2601 mmput(mm); 2602 put_task_struct(usertask); 2603 } 2604 2605 /** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given 2606 * KFD process identified by process_info 2607 * 2608 * @process_info: amdkfd_process_info of the KFD process 2609 * 2610 * After memory eviction, restore thread calls this function. The function 2611 * should be called when the Process is still valid. BO restore involves - 2612 * 2613 * 1. Release old eviction fence and create new one 2614 * 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list. 2615 * 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of 2616 * BOs that need to be reserved. 2617 * 4. Reserve all the BOs 2618 * 5. Validate of PD and PT BOs. 2619 * 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence 2620 * 7. Add fence to all PD and PT BOs. 2621 * 8. Unreserve all BOs 2622 */ 2623 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef) 2624 { 2625 struct amdgpu_bo_list_entry *pd_bo_list; 2626 struct amdkfd_process_info *process_info = info; 2627 struct amdgpu_vm *peer_vm; 2628 struct kgd_mem *mem; 2629 struct bo_vm_reservation_context ctx; 2630 struct amdgpu_amdkfd_fence *new_fence; 2631 int ret = 0, i; 2632 struct list_head duplicate_save; 2633 struct amdgpu_sync sync_obj; 2634 unsigned long failed_size = 0; 2635 unsigned long total_size = 0; 2636 2637 INIT_LIST_HEAD(&duplicate_save); 2638 INIT_LIST_HEAD(&ctx.list); 2639 INIT_LIST_HEAD(&ctx.duplicates); 2640 2641 pd_bo_list = kcalloc(process_info->n_vms, 2642 sizeof(struct amdgpu_bo_list_entry), 2643 GFP_KERNEL); 2644 if (!pd_bo_list) 2645 return -ENOMEM; 2646 2647 i = 0; 2648 mutex_lock(&process_info->lock); 2649 list_for_each_entry(peer_vm, &process_info->vm_list_head, 2650 vm_list_node) 2651 amdgpu_vm_get_pd_bo(peer_vm, &ctx.list, &pd_bo_list[i++]); 2652 2653 /* Reserve all BOs and page tables/directory. Add all BOs from 2654 * kfd_bo_list to ctx.list 2655 */ 2656 list_for_each_entry(mem, &process_info->kfd_bo_list, 2657 validate_list.head) { 2658 2659 list_add_tail(&mem->resv_list.head, &ctx.list); 2660 mem->resv_list.bo = mem->validate_list.bo; 2661 mem->resv_list.num_shared = mem->validate_list.num_shared; 2662 } 2663 2664 ret = ttm_eu_reserve_buffers(&ctx.ticket, &ctx.list, 2665 false, &duplicate_save); 2666 if (ret) { 2667 pr_debug("Memory eviction: TTM Reserve Failed. Try again\n"); 2668 goto ttm_reserve_fail; 2669 } 2670 2671 amdgpu_sync_create(&sync_obj); 2672 2673 /* Validate PDs and PTs */ 2674 ret = process_validate_vms(process_info); 2675 if (ret) 2676 goto validate_map_fail; 2677 2678 ret = process_sync_pds_resv(process_info, &sync_obj); 2679 if (ret) { 2680 pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n"); 2681 goto validate_map_fail; 2682 } 2683 2684 /* Validate BOs and map them to GPUVM (update VM page tables). */ 2685 list_for_each_entry(mem, &process_info->kfd_bo_list, 2686 validate_list.head) { 2687 2688 struct amdgpu_bo *bo = mem->bo; 2689 uint32_t domain = mem->domain; 2690 struct kfd_mem_attachment *attachment; 2691 struct dma_resv_iter cursor; 2692 struct dma_fence *fence; 2693 2694 total_size += amdgpu_bo_size(bo); 2695 2696 ret = amdgpu_amdkfd_bo_validate(bo, domain, false); 2697 if (ret) { 2698 pr_debug("Memory eviction: Validate BOs failed\n"); 2699 failed_size += amdgpu_bo_size(bo); 2700 ret = amdgpu_amdkfd_bo_validate(bo, 2701 AMDGPU_GEM_DOMAIN_GTT, false); 2702 if (ret) { 2703 pr_debug("Memory eviction: Try again\n"); 2704 goto validate_map_fail; 2705 } 2706 } 2707 dma_resv_for_each_fence(&cursor, bo->tbo.base.resv, 2708 DMA_RESV_USAGE_KERNEL, fence) { 2709 ret = amdgpu_sync_fence(&sync_obj, fence); 2710 if (ret) { 2711 pr_debug("Memory eviction: Sync BO fence failed. Try again\n"); 2712 goto validate_map_fail; 2713 } 2714 } 2715 list_for_each_entry(attachment, &mem->attachments, list) { 2716 if (!attachment->is_mapped) 2717 continue; 2718 2719 kfd_mem_dmaunmap_attachment(mem, attachment); 2720 ret = update_gpuvm_pte(mem, attachment, &sync_obj); 2721 if (ret) { 2722 pr_debug("Memory eviction: update PTE failed. Try again\n"); 2723 goto validate_map_fail; 2724 } 2725 } 2726 } 2727 2728 if (failed_size) 2729 pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size); 2730 2731 /* Update page directories */ 2732 ret = process_update_pds(process_info, &sync_obj); 2733 if (ret) { 2734 pr_debug("Memory eviction: update PDs failed. Try again\n"); 2735 goto validate_map_fail; 2736 } 2737 2738 /* Wait for validate and PT updates to finish */ 2739 amdgpu_sync_wait(&sync_obj, false); 2740 2741 /* Release old eviction fence and create new one, because fence only 2742 * goes from unsignaled to signaled, fence cannot be reused. 2743 * Use context and mm from the old fence. 2744 */ 2745 new_fence = amdgpu_amdkfd_fence_create( 2746 process_info->eviction_fence->base.context, 2747 process_info->eviction_fence->mm, 2748 NULL); 2749 if (!new_fence) { 2750 pr_err("Failed to create eviction fence\n"); 2751 ret = -ENOMEM; 2752 goto validate_map_fail; 2753 } 2754 dma_fence_put(&process_info->eviction_fence->base); 2755 process_info->eviction_fence = new_fence; 2756 *ef = dma_fence_get(&new_fence->base); 2757 2758 /* Attach new eviction fence to all BOs except pinned ones */ 2759 list_for_each_entry(mem, &process_info->kfd_bo_list, 2760 validate_list.head) { 2761 if (mem->bo->tbo.pin_count) 2762 continue; 2763 2764 dma_resv_add_fence(mem->bo->tbo.base.resv, 2765 &process_info->eviction_fence->base, 2766 DMA_RESV_USAGE_BOOKKEEP); 2767 } 2768 /* Attach eviction fence to PD / PT BOs */ 2769 list_for_each_entry(peer_vm, &process_info->vm_list_head, 2770 vm_list_node) { 2771 struct amdgpu_bo *bo = peer_vm->root.bo; 2772 2773 dma_resv_add_fence(bo->tbo.base.resv, 2774 &process_info->eviction_fence->base, 2775 DMA_RESV_USAGE_BOOKKEEP); 2776 } 2777 2778 validate_map_fail: 2779 ttm_eu_backoff_reservation(&ctx.ticket, &ctx.list); 2780 amdgpu_sync_free(&sync_obj); 2781 ttm_reserve_fail: 2782 mutex_unlock(&process_info->lock); 2783 kfree(pd_bo_list); 2784 return ret; 2785 } 2786 2787 int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem) 2788 { 2789 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info; 2790 struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws; 2791 int ret; 2792 2793 if (!info || !gws) 2794 return -EINVAL; 2795 2796 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); 2797 if (!*mem) 2798 return -ENOMEM; 2799 2800 mutex_init(&(*mem)->lock); 2801 INIT_LIST_HEAD(&(*mem)->attachments); 2802 (*mem)->bo = amdgpu_bo_ref(gws_bo); 2803 (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS; 2804 (*mem)->process_info = process_info; 2805 add_kgd_mem_to_kfd_bo_list(*mem, process_info, false); 2806 amdgpu_sync_create(&(*mem)->sync); 2807 2808 2809 /* Validate gws bo the first time it is added to process */ 2810 mutex_lock(&(*mem)->process_info->lock); 2811 ret = amdgpu_bo_reserve(gws_bo, false); 2812 if (unlikely(ret)) { 2813 pr_err("Reserve gws bo failed %d\n", ret); 2814 goto bo_reservation_failure; 2815 } 2816 2817 ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true); 2818 if (ret) { 2819 pr_err("GWS BO validate failed %d\n", ret); 2820 goto bo_validation_failure; 2821 } 2822 /* GWS resource is shared b/t amdgpu and amdkfd 2823 * Add process eviction fence to bo so they can 2824 * evict each other. 2825 */ 2826 ret = dma_resv_reserve_fences(gws_bo->tbo.base.resv, 1); 2827 if (ret) 2828 goto reserve_shared_fail; 2829 dma_resv_add_fence(gws_bo->tbo.base.resv, 2830 &process_info->eviction_fence->base, 2831 DMA_RESV_USAGE_BOOKKEEP); 2832 amdgpu_bo_unreserve(gws_bo); 2833 mutex_unlock(&(*mem)->process_info->lock); 2834 2835 return ret; 2836 2837 reserve_shared_fail: 2838 bo_validation_failure: 2839 amdgpu_bo_unreserve(gws_bo); 2840 bo_reservation_failure: 2841 mutex_unlock(&(*mem)->process_info->lock); 2842 amdgpu_sync_free(&(*mem)->sync); 2843 remove_kgd_mem_from_kfd_bo_list(*mem, process_info); 2844 amdgpu_bo_unref(&gws_bo); 2845 mutex_destroy(&(*mem)->lock); 2846 kfree(*mem); 2847 *mem = NULL; 2848 return ret; 2849 } 2850 2851 int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem) 2852 { 2853 int ret; 2854 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info; 2855 struct kgd_mem *kgd_mem = (struct kgd_mem *)mem; 2856 struct amdgpu_bo *gws_bo = kgd_mem->bo; 2857 2858 /* Remove BO from process's validate list so restore worker won't touch 2859 * it anymore 2860 */ 2861 remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info); 2862 2863 ret = amdgpu_bo_reserve(gws_bo, false); 2864 if (unlikely(ret)) { 2865 pr_err("Reserve gws bo failed %d\n", ret); 2866 //TODO add BO back to validate_list? 2867 return ret; 2868 } 2869 amdgpu_amdkfd_remove_eviction_fence(gws_bo, 2870 process_info->eviction_fence); 2871 amdgpu_bo_unreserve(gws_bo); 2872 amdgpu_sync_free(&kgd_mem->sync); 2873 amdgpu_bo_unref(&gws_bo); 2874 mutex_destroy(&kgd_mem->lock); 2875 kfree(mem); 2876 return 0; 2877 } 2878 2879 /* Returns GPU-specific tiling mode information */ 2880 int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev, 2881 struct tile_config *config) 2882 { 2883 config->gb_addr_config = adev->gfx.config.gb_addr_config; 2884 config->tile_config_ptr = adev->gfx.config.tile_mode_array; 2885 config->num_tile_configs = 2886 ARRAY_SIZE(adev->gfx.config.tile_mode_array); 2887 config->macro_tile_config_ptr = 2888 adev->gfx.config.macrotile_mode_array; 2889 config->num_macro_tile_configs = 2890 ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); 2891 2892 /* Those values are not set from GFX9 onwards */ 2893 config->num_banks = adev->gfx.config.num_banks; 2894 config->num_ranks = adev->gfx.config.num_ranks; 2895 2896 return 0; 2897 } 2898 2899 bool amdgpu_amdkfd_bo_mapped_to_dev(struct amdgpu_device *adev, struct kgd_mem *mem) 2900 { 2901 struct kfd_mem_attachment *entry; 2902 2903 list_for_each_entry(entry, &mem->attachments, list) { 2904 if (entry->is_mapped && entry->adev == adev) 2905 return true; 2906 } 2907 return false; 2908 } 2909 2910 #if defined(CONFIG_DEBUG_FS) 2911 2912 int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data) 2913 { 2914 2915 spin_lock(&kfd_mem_limit.mem_limit_lock); 2916 seq_printf(m, "System mem used %lldM out of %lluM\n", 2917 (kfd_mem_limit.system_mem_used >> 20), 2918 (kfd_mem_limit.max_system_mem_limit >> 20)); 2919 seq_printf(m, "TTM mem used %lldM out of %lluM\n", 2920 (kfd_mem_limit.ttm_mem_used >> 20), 2921 (kfd_mem_limit.max_ttm_mem_limit >> 20)); 2922 spin_unlock(&kfd_mem_limit.mem_limit_lock); 2923 2924 return 0; 2925 } 2926 2927 #endif 2928