Lines Matching refs:bo
284 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo) in amdgpu_amdkfd_release_notify() argument
286 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); in amdgpu_amdkfd_release_notify()
287 u32 alloc_flags = bo->kfd_bo->alloc_flags; in amdgpu_amdkfd_release_notify()
288 u64 size = amdgpu_bo_size(bo); in amdgpu_amdkfd_release_notify()
291 bo->xcp_id); in amdgpu_amdkfd_release_notify()
293 kfree(bo->kfd_bo); in amdgpu_amdkfd_release_notify()
313 ret = amdgpu_bo_reserve(mem->bo, false); in create_dmamap_sg_bo()
318 flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT | in create_dmamap_sg_bo()
321 ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1, in create_dmamap_sg_bo()
323 ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0); in create_dmamap_sg_bo()
325 amdgpu_bo_unreserve(mem->bo); in create_dmamap_sg_bo()
333 (*bo_out)->parent = amdgpu_bo_ref(mem->bo); in create_dmamap_sg_bo()
346 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, in amdgpu_amdkfd_remove_eviction_fence() argument
358 dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context, in amdgpu_amdkfd_remove_eviction_fence()
364 int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) in amdgpu_amdkfd_remove_fence_on_pt_pd_bos() argument
366 struct amdgpu_bo *root = bo; in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
392 BUG_ON(!dma_resv_trylock(bo->tbo.base.resv)); in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
393 ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef); in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
394 dma_resv_unlock(bo->tbo.base.resv); in amdgpu_amdkfd_remove_fence_on_pt_pd_bos()
400 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, in amdgpu_amdkfd_bo_validate() argument
406 if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), in amdgpu_amdkfd_bo_validate()
410 amdgpu_bo_placement_from_domain(bo, domain); in amdgpu_amdkfd_bo_validate()
412 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in amdgpu_amdkfd_bo_validate()
416 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); in amdgpu_amdkfd_bo_validate()
422 static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo) in amdgpu_amdkfd_validate_vm_bo() argument
424 return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false); in amdgpu_amdkfd_validate_vm_bo()
436 struct amdgpu_bo *pd = vm->root.bo; in vm_validate_pt_pd_bos()
446 vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo); in vm_validate_pt_pd_bos()
453 struct amdgpu_bo *pd = vm->root.bo; in vm_update_pds()
517 struct amdgpu_bo *bo = attachment->bo_va->base.bo; in kfd_mem_dmamap_userptr() local
519 struct ttm_tt *src_ttm = mem->bo->tbo.ttm; in kfd_mem_dmamap_userptr()
520 struct ttm_tt *ttm = bo->tbo.ttm; in kfd_mem_dmamap_userptr()
542 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); in kfd_mem_dmamap_userptr()
543 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in kfd_mem_dmamap_userptr()
564 struct amdgpu_bo *bo = attachment->bo_va->base.bo; in kfd_mem_dmamap_dmabuf() local
567 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); in kfd_mem_dmamap_dmabuf()
568 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in kfd_mem_dmamap_dmabuf()
572 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); in kfd_mem_dmamap_dmabuf()
573 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in kfd_mem_dmamap_dmabuf()
607 struct amdgpu_bo *bo = attachment->bo_va->base.bo; in kfd_mem_dmamap_sg_bo() local
609 struct ttm_tt *ttm = bo->tbo.ttm; in kfd_mem_dmamap_sg_bo()
624 dma_addr = mem->bo->tbo.sg->sgl->dma_address; in kfd_mem_dmamap_sg_bo()
625 pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length); in kfd_mem_dmamap_sg_bo()
628 mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC); in kfd_mem_dmamap_sg_bo()
634 ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length); in kfd_mem_dmamap_sg_bo()
640 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); in kfd_mem_dmamap_sg_bo()
641 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in kfd_mem_dmamap_sg_bo()
652 dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length, in kfd_mem_dmamap_sg_bo()
684 struct amdgpu_bo *bo = attachment->bo_va->base.bo; in kfd_mem_dmaunmap_userptr() local
686 struct ttm_tt *ttm = bo->tbo.ttm; in kfd_mem_dmaunmap_userptr()
691 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); in kfd_mem_dmaunmap_userptr()
692 ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in kfd_mem_dmaunmap_userptr()
730 struct amdgpu_bo *bo = attachment->bo_va->base.bo; in kfd_mem_dmaunmap_sg_bo() local
732 struct ttm_tt *ttm = bo->tbo.ttm; in kfd_mem_dmaunmap_sg_bo()
740 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); in kfd_mem_dmaunmap_sg_bo()
741 ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in kfd_mem_dmaunmap_sg_bo()
750 bo->tbo.sg = NULL; in kfd_mem_dmaunmap_sg_bo()
778 &mem->bo->tbo.base, in kfd_mem_export_dmabuf()
791 struct amdgpu_bo **bo) in kfd_mem_attach_dmabuf() argument
804 *bo = gem_to_amdgpu_bo(gobj); in kfd_mem_attach_dmabuf()
805 (*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE; in kfd_mem_attach_dmabuf()
826 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); in kfd_mem_attach()
827 unsigned long bo_size = mem->bo->tbo.base.size; in kfd_mem_attach()
830 struct amdgpu_bo *bo[2] = {NULL, NULL}; in kfd_mem_attach() local
868 (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && reuse_dmamap(adev, bo_adev)) || in kfd_mem_attach()
875 bo[i] = mem->bo; in kfd_mem_attach()
876 drm_gem_object_get(&bo[i]->tbo.base); in kfd_mem_attach()
880 bo[i] = bo[0]; in kfd_mem_attach()
881 drm_gem_object_get(&bo[i]->tbo.base); in kfd_mem_attach()
882 } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) { in kfd_mem_attach()
885 ret = create_dmamap_sg_bo(adev, mem, &bo[i]); in kfd_mem_attach()
889 } else if (mem->bo->tbo.type == ttm_bo_type_sg) { in kfd_mem_attach()
894 ret = create_dmamap_sg_bo(adev, mem, &bo[i]); in kfd_mem_attach()
901 ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]); in kfd_mem_attach()
912 ret = amdgpu_bo_reserve(bo[i], false); in kfd_mem_attach()
917 attachment[i]->bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]); in kfd_mem_attach()
918 amdgpu_bo_unreserve(bo[i]); in kfd_mem_attach()
940 amdgpu_bo_reserve(bo[i], true); in kfd_mem_attach()
942 amdgpu_bo_unreserve(bo[i]); in kfd_mem_attach()
945 if (bo[i]) in kfd_mem_attach()
946 drm_gem_object_put(&bo[i]->tbo.base); in kfd_mem_attach()
954 struct amdgpu_bo *bo = attachment->bo_va->base.bo; in kfd_mem_detach() local
959 drm_gem_object_put(&bo->tbo.base); in kfd_mem_detach()
1001 struct amdgpu_bo *bo = mem->bo; in init_user_pages() local
1008 ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0); in init_user_pages()
1014 ret = amdgpu_hmm_register(bo, user_addr); in init_user_pages()
1035 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages, &range); in init_user_pages()
1041 ret = amdgpu_bo_reserve(bo, true); in init_user_pages()
1046 amdgpu_bo_placement_from_domain(bo, mem->domain); in init_user_pages()
1047 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in init_user_pages()
1050 amdgpu_bo_unreserve(bo); in init_user_pages()
1053 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range); in init_user_pages()
1056 amdgpu_hmm_unregister(bo); in init_user_pages()
1092 struct amdgpu_bo *bo = mem->bo; in reserve_bo_and_vm() local
1106 ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1); in reserve_bo_and_vm()
1134 struct amdgpu_bo *bo = mem->bo; in reserve_bo_and_cond_vms() local
1156 ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1); in reserve_bo_and_cond_vms()
1240 amdgpu_bo_size(entry->bo_va->base.bo), in map_bo_to_gpuvm()
1287 struct amdgpu_bo *pd = peer_vm->root.bo; in process_sync_pds_resv()
1354 ret = amdgpu_bo_reserve(vm->root.bo, true); in init_kfd_vm()
1362 ret = amdgpu_bo_sync_wait(vm->root.bo, in init_kfd_vm()
1366 ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1); in init_kfd_vm()
1369 dma_resv_add_fence(vm->root.bo->tbo.base.resv, in init_kfd_vm()
1372 amdgpu_bo_unreserve(vm->root.bo); in init_kfd_vm()
1386 amdgpu_bo_unreserve(vm->root.bo); in init_kfd_vm()
1415 static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain) in amdgpu_amdkfd_gpuvm_pin_bo() argument
1419 ret = amdgpu_bo_reserve(bo, false); in amdgpu_amdkfd_gpuvm_pin_bo()
1423 ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0); in amdgpu_amdkfd_gpuvm_pin_bo()
1427 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); in amdgpu_amdkfd_gpuvm_pin_bo()
1428 amdgpu_bo_unreserve(bo); in amdgpu_amdkfd_gpuvm_pin_bo()
1441 static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo) in amdgpu_amdkfd_gpuvm_unpin_bo() argument
1445 ret = amdgpu_bo_reserve(bo, false); in amdgpu_amdkfd_gpuvm_unpin_bo()
1449 amdgpu_bo_unpin(bo); in amdgpu_amdkfd_gpuvm_unpin_bo()
1450 amdgpu_bo_unreserve(bo); in amdgpu_amdkfd_gpuvm_unpin_bo()
1555 struct amdgpu_bo *pd = avm->root.bo; in amdgpu_amdkfd_gpuvm_get_process_page_dir()
1642 struct amdgpu_bo *bo; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu() local
1741 bo = gem_to_amdgpu_bo(gobj); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1743 bo->tbo.sg = sg; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1744 bo->tbo.ttm->sg = sg; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1746 bo->kfd_bo = *mem; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1747 (*mem)->bo = bo; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1749 bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1765 ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1770 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1771 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1775 *offset = amdgpu_bo_mmap_offset(bo); in amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu()
1808 unsigned long bo_size = mem->bo->tbo.base.size; in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1809 bool use_release_notifier = (mem->bo->kfd_bo == mem); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1822 amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1844 if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) { in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1845 amdgpu_hmm_unregister(mem->bo); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1847 amdgpu_ttm_tt_discard_user_pages(mem->bo->tbo.ttm, mem->range); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1859 amdgpu_amdkfd_remove_eviction_fence(mem->bo, in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1876 if (mem->bo->tbo.sg) { in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1877 sg_free_table(mem->bo->tbo.sg); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1878 kfree(mem->bo->tbo.sg); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1887 (mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM || in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1889 mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT))) in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1896 drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1905 drm_gem_object_put(&mem->bo->tbo.base); in amdgpu_amdkfd_gpuvm_free_memory_of_gpu()
1923 struct amdgpu_bo *bo; in amdgpu_amdkfd_gpuvm_map_memory_to_gpu() local
1930 bo = mem->bo; in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1931 if (!bo) { in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1946 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1955 bo_size = bo->tbo.base.size; in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1977 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1978 bo->tbo.resource->mem_type == TTM_PL_SYSTEM) in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1986 !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
1991 ret = amdgpu_amdkfd_bo_validate(bo, domain, true); in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
2024 if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->tbo.pin_count) in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
2025 dma_resv_add_fence(bo->tbo.base.resv, in amdgpu_amdkfd_gpuvm_map_memory_to_gpu()
2045 unsigned long bo_size = mem->bo->tbo.base.size; in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
2089 !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
2090 !mem->bo->tbo.pin_count) in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
2091 amdgpu_amdkfd_remove_eviction_fence(mem->bo, in amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu()
2126 int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_device *adev, struct amdgpu_bo *bo) in amdgpu_amdkfd_map_gtt_bo_to_gart() argument
2130 ret = amdgpu_bo_reserve(bo, true); in amdgpu_amdkfd_map_gtt_bo_to_gart()
2136 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); in amdgpu_amdkfd_map_gtt_bo_to_gart()
2142 ret = amdgpu_ttm_alloc_gart(&bo->tbo); in amdgpu_amdkfd_map_gtt_bo_to_gart()
2149 bo, bo->vm_bo->vm->process_info->eviction_fence); in amdgpu_amdkfd_map_gtt_bo_to_gart()
2151 amdgpu_bo_unreserve(bo); in amdgpu_amdkfd_map_gtt_bo_to_gart()
2153 bo = amdgpu_bo_ref(bo); in amdgpu_amdkfd_map_gtt_bo_to_gart()
2158 amdgpu_bo_unpin(bo); in amdgpu_amdkfd_map_gtt_bo_to_gart()
2160 amdgpu_bo_unreserve(bo); in amdgpu_amdkfd_map_gtt_bo_to_gart()
2183 struct amdgpu_bo *bo = mem->bo; in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() local
2185 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2192 ret = amdgpu_bo_reserve(bo, true); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2198 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2204 ret = amdgpu_bo_kmap(bo, kptr); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2211 bo, mem->process_info->eviction_fence); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2214 *size = amdgpu_bo_size(bo); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2216 amdgpu_bo_unreserve(bo); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2222 amdgpu_bo_unpin(bo); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2224 amdgpu_bo_unreserve(bo); in amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel()
2241 struct amdgpu_bo *bo = mem->bo; in amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel() local
2243 amdgpu_bo_reserve(bo, true); in amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel()
2244 amdgpu_bo_kunmap(bo); in amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel()
2245 amdgpu_bo_unpin(bo); in amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel()
2246 amdgpu_bo_unreserve(bo); in amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel()
2268 struct amdgpu_bo *bo; in amdgpu_amdkfd_gpuvm_import_dmabuf() local
2275 bo = gem_to_amdgpu_bo(obj); in amdgpu_amdkfd_gpuvm_import_dmabuf()
2276 if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | in amdgpu_amdkfd_gpuvm_import_dmabuf()
2294 *size = amdgpu_bo_size(bo); in amdgpu_amdkfd_gpuvm_import_dmabuf()
2297 *mmap_offset = amdgpu_bo_mmap_offset(bo); in amdgpu_amdkfd_gpuvm_import_dmabuf()
2303 ((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? in amdgpu_amdkfd_gpuvm_import_dmabuf()
2310 (*mem)->bo = bo; in amdgpu_amdkfd_gpuvm_import_dmabuf()
2312 (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) && !adev->gmc.is_app_apu ? in amdgpu_amdkfd_gpuvm_import_dmabuf()
2397 struct amdgpu_bo *bo; in update_invalid_user_pages() local
2422 bo = mem->bo; in update_invalid_user_pages()
2424 amdgpu_ttm_tt_discard_user_pages(bo->tbo.ttm, mem->range); in update_invalid_user_pages()
2435 if (bo->tbo.resource->mem_type != TTM_PL_SYSTEM) { in update_invalid_user_pages()
2436 if (amdgpu_bo_reserve(bo, true)) in update_invalid_user_pages()
2438 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); in update_invalid_user_pages()
2439 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in update_invalid_user_pages()
2440 amdgpu_bo_unreserve(bo); in update_invalid_user_pages()
2449 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages, in update_invalid_user_pages()
2500 struct amdgpu_bo *bo; in validate_invalid_user_pages() local
2522 gobj = &mem->bo->tbo.base; in validate_invalid_user_pages()
2540 bo = mem->bo; in validate_invalid_user_pages()
2543 if (bo->tbo.ttm->pages[0]) { in validate_invalid_user_pages()
2544 amdgpu_bo_placement_from_domain(bo, mem->domain); in validate_invalid_user_pages()
2545 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in validate_invalid_user_pages()
2606 mem->bo->tbo.ttm, mem->range); in confirm_valid_user_pages_locked()
2763 gobj = &mem->bo->tbo.base; in amdgpu_amdkfd_gpuvm_restore_process_bos()
2788 struct amdgpu_bo *bo = mem->bo; in amdgpu_amdkfd_gpuvm_restore_process_bos() local
2794 total_size += amdgpu_bo_size(bo); in amdgpu_amdkfd_gpuvm_restore_process_bos()
2796 ret = amdgpu_amdkfd_bo_validate(bo, domain, false); in amdgpu_amdkfd_gpuvm_restore_process_bos()
2799 failed_size += amdgpu_bo_size(bo); in amdgpu_amdkfd_gpuvm_restore_process_bos()
2800 ret = amdgpu_amdkfd_bo_validate(bo, in amdgpu_amdkfd_gpuvm_restore_process_bos()
2807 dma_resv_for_each_fence(&cursor, bo->tbo.base.resv, in amdgpu_amdkfd_gpuvm_restore_process_bos()
2819 if (attachment->bo_va->base.bo->tbo.pin_count) in amdgpu_amdkfd_gpuvm_restore_process_bos()
2863 if (mem->bo->tbo.pin_count) in amdgpu_amdkfd_gpuvm_restore_process_bos()
2866 dma_resv_add_fence(mem->bo->tbo.base.resv, in amdgpu_amdkfd_gpuvm_restore_process_bos()
2873 struct amdgpu_bo *bo = peer_vm->root.bo; in amdgpu_amdkfd_gpuvm_restore_process_bos() local
2875 dma_resv_add_fence(bo->tbo.base.resv, in amdgpu_amdkfd_gpuvm_restore_process_bos()
2903 (*mem)->bo = amdgpu_bo_ref(gws_bo); in amdgpu_amdkfd_add_gws_to_process()
2957 struct amdgpu_bo *gws_bo = kgd_mem->bo; in amdgpu_amdkfd_remove_gws_from_process()