1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <linux/dma-fence-array.h> 29 #include <linux/interval_tree_generic.h> 30 #include <linux/idr.h> 31 #include <drm/drmP.h> 32 #include <drm/amdgpu_drm.h> 33 #include "amdgpu.h" 34 #include "amdgpu_trace.h" 35 #include "amdgpu_amdkfd.h" 36 37 /* 38 * GPUVM 39 * GPUVM is similar to the legacy gart on older asics, however 40 * rather than there being a single global gart table 41 * for the entire GPU, there are multiple VM page tables active 42 * at any given time. The VM page tables can contain a mix 43 * vram pages and system memory pages and system memory pages 44 * can be mapped as snooped (cached system pages) or unsnooped 45 * (uncached system pages). 46 * Each VM has an ID associated with it and there is a page table 47 * associated with each VMID. When execting a command buffer, 48 * the kernel tells the the ring what VMID to use for that command 49 * buffer. VMIDs are allocated dynamically as commands are submitted. 50 * The userspace drivers maintain their own address space and the kernel 51 * sets up their pages tables accordingly when they submit their 52 * command buffers and a VMID is assigned. 53 * Cayman/Trinity support up to 8 active VMs at any given time; 54 * SI supports 16. 55 */ 56 57 #define START(node) ((node)->start) 58 #define LAST(node) ((node)->last) 59 60 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, 61 START, LAST, static, amdgpu_vm_it) 62 63 #undef START 64 #undef LAST 65 66 /* Local structure. Encapsulate some VM table update parameters to reduce 67 * the number of function parameters 68 */ 69 struct amdgpu_pte_update_params { 70 /* amdgpu device we do this update for */ 71 struct amdgpu_device *adev; 72 /* optional amdgpu_vm we do this update for */ 73 struct amdgpu_vm *vm; 74 /* address where to copy page table entries from */ 75 uint64_t src; 76 /* indirect buffer to fill with commands */ 77 struct amdgpu_ib *ib; 78 /* Function which actually does the update */ 79 void (*func)(struct amdgpu_pte_update_params *params, 80 struct amdgpu_bo *bo, uint64_t pe, 81 uint64_t addr, unsigned count, uint32_t incr, 82 uint64_t flags); 83 /* The next two are used during VM update by CPU 84 * DMA addresses to use for mapping 85 * Kernel pointer of PD/PT BO that needs to be updated 86 */ 87 dma_addr_t *pages_addr; 88 void *kptr; 89 }; 90 91 /* Helper to disable partial resident texture feature from a fence callback */ 92 struct amdgpu_prt_cb { 93 struct amdgpu_device *adev; 94 struct dma_fence_cb cb; 95 }; 96 97 static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, 98 struct amdgpu_vm *vm, 99 struct amdgpu_bo *bo) 100 { 101 base->vm = vm; 102 base->bo = bo; 103 INIT_LIST_HEAD(&base->bo_list); 104 INIT_LIST_HEAD(&base->vm_status); 105 106 if (!bo) 107 return; 108 list_add_tail(&base->bo_list, &bo->va); 109 110 if (bo->tbo.type == ttm_bo_type_kernel) 111 list_move(&base->vm_status, &vm->relocated); 112 113 if (bo->tbo.resv != vm->root.base.bo->tbo.resv) 114 return; 115 116 if (bo->preferred_domains & 117 amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type)) 118 return; 119 120 /* 121 * we checked all the prerequisites, but it looks like this per vm bo 122 * is currently evicted. add the bo to the evicted list to make sure it 123 * is validated on next vm use to avoid fault. 124 * */ 125 list_move_tail(&base->vm_status, &vm->evicted); 126 } 127 128 /** 129 * amdgpu_vm_level_shift - return the addr shift for each level 130 * 131 * @adev: amdgpu_device pointer 132 * 133 * Returns the number of bits the pfn needs to be right shifted for a level. 134 */ 135 static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev, 136 unsigned level) 137 { 138 unsigned shift = 0xff; 139 140 switch (level) { 141 case AMDGPU_VM_PDB2: 142 case AMDGPU_VM_PDB1: 143 case AMDGPU_VM_PDB0: 144 shift = 9 * (AMDGPU_VM_PDB0 - level) + 145 adev->vm_manager.block_size; 146 break; 147 case AMDGPU_VM_PTB: 148 shift = 0; 149 break; 150 default: 151 dev_err(adev->dev, "the level%d isn't supported.\n", level); 152 } 153 154 return shift; 155 } 156 157 /** 158 * amdgpu_vm_num_entries - return the number of entries in a PD/PT 159 * 160 * @adev: amdgpu_device pointer 161 * 162 * Calculate the number of entries in a page directory or page table. 163 */ 164 static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev, 165 unsigned level) 166 { 167 unsigned shift = amdgpu_vm_level_shift(adev, 168 adev->vm_manager.root_level); 169 170 if (level == adev->vm_manager.root_level) 171 /* For the root directory */ 172 return round_up(adev->vm_manager.max_pfn, 1 << shift) >> shift; 173 else if (level != AMDGPU_VM_PTB) 174 /* Everything in between */ 175 return 512; 176 else 177 /* For the page tables on the leaves */ 178 return AMDGPU_VM_PTE_COUNT(adev); 179 } 180 181 /** 182 * amdgpu_vm_bo_size - returns the size of the BOs in bytes 183 * 184 * @adev: amdgpu_device pointer 185 * 186 * Calculate the size of the BO for a page directory or page table in bytes. 187 */ 188 static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level) 189 { 190 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8); 191 } 192 193 /** 194 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list 195 * 196 * @vm: vm providing the BOs 197 * @validated: head of validation list 198 * @entry: entry to add 199 * 200 * Add the page directory to the list of BOs to 201 * validate for command submission. 202 */ 203 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, 204 struct list_head *validated, 205 struct amdgpu_bo_list_entry *entry) 206 { 207 entry->robj = vm->root.base.bo; 208 entry->priority = 0; 209 entry->tv.bo = &entry->robj->tbo; 210 entry->tv.shared = true; 211 entry->user_pages = NULL; 212 list_add(&entry->tv.head, validated); 213 } 214 215 /** 216 * amdgpu_vm_validate_pt_bos - validate the page table BOs 217 * 218 * @adev: amdgpu device pointer 219 * @vm: vm providing the BOs 220 * @validate: callback to do the validation 221 * @param: parameter for the validation callback 222 * 223 * Validate the page table BOs on command submission if neccessary. 224 */ 225 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, 226 int (*validate)(void *p, struct amdgpu_bo *bo), 227 void *param) 228 { 229 struct ttm_bo_global *glob = adev->mman.bdev.glob; 230 struct amdgpu_vm_bo_base *bo_base, *tmp; 231 int r = 0; 232 233 list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) { 234 struct amdgpu_bo *bo = bo_base->bo; 235 236 if (bo->parent) { 237 r = validate(param, bo); 238 if (r) 239 break; 240 241 spin_lock(&glob->lru_lock); 242 ttm_bo_move_to_lru_tail(&bo->tbo); 243 if (bo->shadow) 244 ttm_bo_move_to_lru_tail(&bo->shadow->tbo); 245 spin_unlock(&glob->lru_lock); 246 } 247 248 if (bo->tbo.type != ttm_bo_type_kernel) { 249 spin_lock(&vm->moved_lock); 250 list_move(&bo_base->vm_status, &vm->moved); 251 spin_unlock(&vm->moved_lock); 252 } else { 253 list_move(&bo_base->vm_status, &vm->relocated); 254 } 255 } 256 257 spin_lock(&glob->lru_lock); 258 list_for_each_entry(bo_base, &vm->idle, vm_status) { 259 struct amdgpu_bo *bo = bo_base->bo; 260 261 if (!bo->parent) 262 continue; 263 264 ttm_bo_move_to_lru_tail(&bo->tbo); 265 if (bo->shadow) 266 ttm_bo_move_to_lru_tail(&bo->shadow->tbo); 267 } 268 spin_unlock(&glob->lru_lock); 269 270 return r; 271 } 272 273 /** 274 * amdgpu_vm_ready - check VM is ready for updates 275 * 276 * @vm: VM to check 277 * 278 * Check if all VM PDs/PTs are ready for updates 279 */ 280 bool amdgpu_vm_ready(struct amdgpu_vm *vm) 281 { 282 return list_empty(&vm->evicted); 283 } 284 285 /** 286 * amdgpu_vm_clear_bo - initially clear the PDs/PTs 287 * 288 * @adev: amdgpu_device pointer 289 * @bo: BO to clear 290 * @level: level this BO is at 291 * 292 * Root PD needs to be reserved when calling this. 293 */ 294 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, 295 struct amdgpu_vm *vm, struct amdgpu_bo *bo, 296 unsigned level, bool pte_support_ats) 297 { 298 struct ttm_operation_ctx ctx = { true, false }; 299 struct dma_fence *fence = NULL; 300 unsigned entries, ats_entries; 301 struct amdgpu_ring *ring; 302 struct amdgpu_job *job; 303 uint64_t addr; 304 int r; 305 306 addr = amdgpu_bo_gpu_offset(bo); 307 entries = amdgpu_bo_size(bo) / 8; 308 309 if (pte_support_ats) { 310 if (level == adev->vm_manager.root_level) { 311 ats_entries = amdgpu_vm_level_shift(adev, level); 312 ats_entries += AMDGPU_GPU_PAGE_SHIFT; 313 ats_entries = AMDGPU_VA_HOLE_START >> ats_entries; 314 ats_entries = min(ats_entries, entries); 315 entries -= ats_entries; 316 } else { 317 ats_entries = entries; 318 entries = 0; 319 } 320 } else { 321 ats_entries = 0; 322 } 323 324 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched); 325 326 r = reservation_object_reserve_shared(bo->tbo.resv); 327 if (r) 328 return r; 329 330 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 331 if (r) 332 goto error; 333 334 r = amdgpu_job_alloc_with_ib(adev, 64, &job); 335 if (r) 336 goto error; 337 338 if (ats_entries) { 339 uint64_t ats_value; 340 341 ats_value = AMDGPU_PTE_DEFAULT_ATC; 342 if (level != AMDGPU_VM_PTB) 343 ats_value |= AMDGPU_PDE_PTE; 344 345 amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0, 346 ats_entries, 0, ats_value); 347 addr += ats_entries * 8; 348 } 349 350 if (entries) 351 amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0, 352 entries, 0, 0); 353 354 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 355 356 WARN_ON(job->ibs[0].length_dw > 64); 357 r = amdgpu_sync_resv(adev, &job->sync, bo->tbo.resv, 358 AMDGPU_FENCE_OWNER_UNDEFINED, false); 359 if (r) 360 goto error_free; 361 362 r = amdgpu_job_submit(job, ring, &vm->entity, 363 AMDGPU_FENCE_OWNER_UNDEFINED, &fence); 364 if (r) 365 goto error_free; 366 367 amdgpu_bo_fence(bo, fence, true); 368 dma_fence_put(fence); 369 370 if (bo->shadow) 371 return amdgpu_vm_clear_bo(adev, vm, bo->shadow, 372 level, pte_support_ats); 373 374 return 0; 375 376 error_free: 377 amdgpu_job_free(job); 378 379 error: 380 return r; 381 } 382 383 /** 384 * amdgpu_vm_alloc_levels - allocate the PD/PT levels 385 * 386 * @adev: amdgpu_device pointer 387 * @vm: requested vm 388 * @saddr: start of the address range 389 * @eaddr: end of the address range 390 * 391 * Make sure the page directories and page tables are allocated 392 */ 393 static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev, 394 struct amdgpu_vm *vm, 395 struct amdgpu_vm_pt *parent, 396 uint64_t saddr, uint64_t eaddr, 397 unsigned level, bool ats) 398 { 399 unsigned shift = amdgpu_vm_level_shift(adev, level); 400 unsigned pt_idx, from, to; 401 u64 flags; 402 int r; 403 404 if (!parent->entries) { 405 unsigned num_entries = amdgpu_vm_num_entries(adev, level); 406 407 parent->entries = kvmalloc_array(num_entries, 408 sizeof(struct amdgpu_vm_pt), 409 GFP_KERNEL | __GFP_ZERO); 410 if (!parent->entries) 411 return -ENOMEM; 412 memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt)); 413 } 414 415 from = saddr >> shift; 416 to = eaddr >> shift; 417 if (from >= amdgpu_vm_num_entries(adev, level) || 418 to >= amdgpu_vm_num_entries(adev, level)) 419 return -EINVAL; 420 421 ++level; 422 saddr = saddr & ((1 << shift) - 1); 423 eaddr = eaddr & ((1 << shift) - 1); 424 425 flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 426 if (vm->use_cpu_for_update) 427 flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 428 else 429 flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS | 430 AMDGPU_GEM_CREATE_SHADOW); 431 432 /* walk over the address space and allocate the page tables */ 433 for (pt_idx = from; pt_idx <= to; ++pt_idx) { 434 struct reservation_object *resv = vm->root.base.bo->tbo.resv; 435 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; 436 struct amdgpu_bo *pt; 437 438 if (!entry->base.bo) { 439 struct amdgpu_bo_param bp; 440 441 memset(&bp, 0, sizeof(bp)); 442 bp.size = amdgpu_vm_bo_size(adev, level); 443 bp.byte_align = AMDGPU_GPU_PAGE_SIZE; 444 bp.domain = AMDGPU_GEM_DOMAIN_VRAM; 445 bp.flags = flags; 446 bp.type = ttm_bo_type_kernel; 447 bp.resv = resv; 448 r = amdgpu_bo_create(adev, &bp, &pt); 449 if (r) 450 return r; 451 452 r = amdgpu_vm_clear_bo(adev, vm, pt, level, ats); 453 if (r) { 454 amdgpu_bo_unref(&pt->shadow); 455 amdgpu_bo_unref(&pt); 456 return r; 457 } 458 459 if (vm->use_cpu_for_update) { 460 r = amdgpu_bo_kmap(pt, NULL); 461 if (r) { 462 amdgpu_bo_unref(&pt->shadow); 463 amdgpu_bo_unref(&pt); 464 return r; 465 } 466 } 467 468 /* Keep a reference to the root directory to avoid 469 * freeing them up in the wrong order. 470 */ 471 pt->parent = amdgpu_bo_ref(parent->base.bo); 472 473 amdgpu_vm_bo_base_init(&entry->base, vm, pt); 474 } 475 476 if (level < AMDGPU_VM_PTB) { 477 uint64_t sub_saddr = (pt_idx == from) ? saddr : 0; 478 uint64_t sub_eaddr = (pt_idx == to) ? eaddr : 479 ((1 << shift) - 1); 480 r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr, 481 sub_eaddr, level, ats); 482 if (r) 483 return r; 484 } 485 } 486 487 return 0; 488 } 489 490 /** 491 * amdgpu_vm_alloc_pts - Allocate page tables. 492 * 493 * @adev: amdgpu_device pointer 494 * @vm: VM to allocate page tables for 495 * @saddr: Start address which needs to be allocated 496 * @size: Size from start address we need. 497 * 498 * Make sure the page tables are allocated. 499 */ 500 int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, 501 struct amdgpu_vm *vm, 502 uint64_t saddr, uint64_t size) 503 { 504 uint64_t eaddr; 505 bool ats = false; 506 507 /* validate the parameters */ 508 if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK) 509 return -EINVAL; 510 511 eaddr = saddr + size - 1; 512 513 if (vm->pte_support_ats) 514 ats = saddr < AMDGPU_VA_HOLE_START; 515 516 saddr /= AMDGPU_GPU_PAGE_SIZE; 517 eaddr /= AMDGPU_GPU_PAGE_SIZE; 518 519 if (eaddr >= adev->vm_manager.max_pfn) { 520 dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n", 521 eaddr, adev->vm_manager.max_pfn); 522 return -EINVAL; 523 } 524 525 return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 526 adev->vm_manager.root_level, ats); 527 } 528 529 /** 530 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug 531 * 532 * @adev: amdgpu_device pointer 533 */ 534 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev) 535 { 536 const struct amdgpu_ip_block *ip_block; 537 bool has_compute_vm_bug; 538 struct amdgpu_ring *ring; 539 int i; 540 541 has_compute_vm_bug = false; 542 543 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); 544 if (ip_block) { 545 /* Compute has a VM bug for GFX version < 7. 546 Compute has a VM bug for GFX 8 MEC firmware version < 673.*/ 547 if (ip_block->version->major <= 7) 548 has_compute_vm_bug = true; 549 else if (ip_block->version->major == 8) 550 if (adev->gfx.mec_fw_version < 673) 551 has_compute_vm_bug = true; 552 } 553 554 for (i = 0; i < adev->num_rings; i++) { 555 ring = adev->rings[i]; 556 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 557 /* only compute rings */ 558 ring->has_compute_vm_bug = has_compute_vm_bug; 559 else 560 ring->has_compute_vm_bug = false; 561 } 562 } 563 564 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring, 565 struct amdgpu_job *job) 566 { 567 struct amdgpu_device *adev = ring->adev; 568 unsigned vmhub = ring->funcs->vmhub; 569 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 570 struct amdgpu_vmid *id; 571 bool gds_switch_needed; 572 bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug; 573 574 if (job->vmid == 0) 575 return false; 576 id = &id_mgr->ids[job->vmid]; 577 gds_switch_needed = ring->funcs->emit_gds_switch && ( 578 id->gds_base != job->gds_base || 579 id->gds_size != job->gds_size || 580 id->gws_base != job->gws_base || 581 id->gws_size != job->gws_size || 582 id->oa_base != job->oa_base || 583 id->oa_size != job->oa_size); 584 585 if (amdgpu_vmid_had_gpu_reset(adev, id)) 586 return true; 587 588 return vm_flush_needed || gds_switch_needed; 589 } 590 591 static bool amdgpu_vm_is_large_bar(struct amdgpu_device *adev) 592 { 593 return (adev->gmc.real_vram_size == adev->gmc.visible_vram_size); 594 } 595 596 /** 597 * amdgpu_vm_flush - hardware flush the vm 598 * 599 * @ring: ring to use for flush 600 * @vmid: vmid number to use 601 * @pd_addr: address of the page directory 602 * 603 * Emit a VM flush when it is necessary. 604 */ 605 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync) 606 { 607 struct amdgpu_device *adev = ring->adev; 608 unsigned vmhub = ring->funcs->vmhub; 609 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 610 struct amdgpu_vmid *id = &id_mgr->ids[job->vmid]; 611 bool gds_switch_needed = ring->funcs->emit_gds_switch && ( 612 id->gds_base != job->gds_base || 613 id->gds_size != job->gds_size || 614 id->gws_base != job->gws_base || 615 id->gws_size != job->gws_size || 616 id->oa_base != job->oa_base || 617 id->oa_size != job->oa_size); 618 bool vm_flush_needed = job->vm_needs_flush; 619 bool pasid_mapping_needed = id->pasid != job->pasid || 620 !id->pasid_mapping || 621 !dma_fence_is_signaled(id->pasid_mapping); 622 struct dma_fence *fence = NULL; 623 unsigned patch_offset = 0; 624 int r; 625 626 if (amdgpu_vmid_had_gpu_reset(adev, id)) { 627 gds_switch_needed = true; 628 vm_flush_needed = true; 629 pasid_mapping_needed = true; 630 } 631 632 gds_switch_needed &= !!ring->funcs->emit_gds_switch; 633 vm_flush_needed &= !!ring->funcs->emit_vm_flush; 634 pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping && 635 ring->funcs->emit_wreg; 636 637 if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync) 638 return 0; 639 640 if (ring->funcs->init_cond_exec) 641 patch_offset = amdgpu_ring_init_cond_exec(ring); 642 643 if (need_pipe_sync) 644 amdgpu_ring_emit_pipeline_sync(ring); 645 646 if (vm_flush_needed) { 647 trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr); 648 amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr); 649 } 650 651 if (pasid_mapping_needed) 652 amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid); 653 654 if (vm_flush_needed || pasid_mapping_needed) { 655 r = amdgpu_fence_emit(ring, &fence, 0); 656 if (r) 657 return r; 658 } 659 660 if (vm_flush_needed) { 661 mutex_lock(&id_mgr->lock); 662 dma_fence_put(id->last_flush); 663 id->last_flush = dma_fence_get(fence); 664 id->current_gpu_reset_count = 665 atomic_read(&adev->gpu_reset_counter); 666 mutex_unlock(&id_mgr->lock); 667 } 668 669 if (pasid_mapping_needed) { 670 id->pasid = job->pasid; 671 dma_fence_put(id->pasid_mapping); 672 id->pasid_mapping = dma_fence_get(fence); 673 } 674 dma_fence_put(fence); 675 676 if (ring->funcs->emit_gds_switch && gds_switch_needed) { 677 id->gds_base = job->gds_base; 678 id->gds_size = job->gds_size; 679 id->gws_base = job->gws_base; 680 id->gws_size = job->gws_size; 681 id->oa_base = job->oa_base; 682 id->oa_size = job->oa_size; 683 amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base, 684 job->gds_size, job->gws_base, 685 job->gws_size, job->oa_base, 686 job->oa_size); 687 } 688 689 if (ring->funcs->patch_cond_exec) 690 amdgpu_ring_patch_cond_exec(ring, patch_offset); 691 692 /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */ 693 if (ring->funcs->emit_switch_buffer) { 694 amdgpu_ring_emit_switch_buffer(ring); 695 amdgpu_ring_emit_switch_buffer(ring); 696 } 697 return 0; 698 } 699 700 /** 701 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo 702 * 703 * @vm: requested vm 704 * @bo: requested buffer object 705 * 706 * Find @bo inside the requested vm. 707 * Search inside the @bos vm list for the requested vm 708 * Returns the found bo_va or NULL if none is found 709 * 710 * Object has to be reserved! 711 */ 712 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, 713 struct amdgpu_bo *bo) 714 { 715 struct amdgpu_bo_va *bo_va; 716 717 list_for_each_entry(bo_va, &bo->va, base.bo_list) { 718 if (bo_va->base.vm == vm) { 719 return bo_va; 720 } 721 } 722 return NULL; 723 } 724 725 /** 726 * amdgpu_vm_do_set_ptes - helper to call the right asic function 727 * 728 * @params: see amdgpu_pte_update_params definition 729 * @bo: PD/PT to update 730 * @pe: addr of the page entry 731 * @addr: dst addr to write into pe 732 * @count: number of page entries to update 733 * @incr: increase next addr by incr bytes 734 * @flags: hw access flags 735 * 736 * Traces the parameters and calls the right asic functions 737 * to setup the page table using the DMA. 738 */ 739 static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params, 740 struct amdgpu_bo *bo, 741 uint64_t pe, uint64_t addr, 742 unsigned count, uint32_t incr, 743 uint64_t flags) 744 { 745 pe += amdgpu_bo_gpu_offset(bo); 746 trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags); 747 748 if (count < 3) { 749 amdgpu_vm_write_pte(params->adev, params->ib, pe, 750 addr | flags, count, incr); 751 752 } else { 753 amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr, 754 count, incr, flags); 755 } 756 } 757 758 /** 759 * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART 760 * 761 * @params: see amdgpu_pte_update_params definition 762 * @bo: PD/PT to update 763 * @pe: addr of the page entry 764 * @addr: dst addr to write into pe 765 * @count: number of page entries to update 766 * @incr: increase next addr by incr bytes 767 * @flags: hw access flags 768 * 769 * Traces the parameters and calls the DMA function to copy the PTEs. 770 */ 771 static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params, 772 struct amdgpu_bo *bo, 773 uint64_t pe, uint64_t addr, 774 unsigned count, uint32_t incr, 775 uint64_t flags) 776 { 777 uint64_t src = (params->src + (addr >> 12) * 8); 778 779 pe += amdgpu_bo_gpu_offset(bo); 780 trace_amdgpu_vm_copy_ptes(pe, src, count); 781 782 amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count); 783 } 784 785 /** 786 * amdgpu_vm_map_gart - Resolve gart mapping of addr 787 * 788 * @pages_addr: optional DMA address to use for lookup 789 * @addr: the unmapped addr 790 * 791 * Look up the physical address of the page that the pte resolves 792 * to and return the pointer for the page table entry. 793 */ 794 static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr) 795 { 796 uint64_t result; 797 798 /* page table offset */ 799 result = pages_addr[addr >> PAGE_SHIFT]; 800 801 /* in case cpu page size != gpu page size*/ 802 result |= addr & (~PAGE_MASK); 803 804 result &= 0xFFFFFFFFFFFFF000ULL; 805 806 return result; 807 } 808 809 /** 810 * amdgpu_vm_cpu_set_ptes - helper to update page tables via CPU 811 * 812 * @params: see amdgpu_pte_update_params definition 813 * @bo: PD/PT to update 814 * @pe: kmap addr of the page entry 815 * @addr: dst addr to write into pe 816 * @count: number of page entries to update 817 * @incr: increase next addr by incr bytes 818 * @flags: hw access flags 819 * 820 * Write count number of PT/PD entries directly. 821 */ 822 static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params, 823 struct amdgpu_bo *bo, 824 uint64_t pe, uint64_t addr, 825 unsigned count, uint32_t incr, 826 uint64_t flags) 827 { 828 unsigned int i; 829 uint64_t value; 830 831 pe += (unsigned long)amdgpu_bo_kptr(bo); 832 833 trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags); 834 835 for (i = 0; i < count; i++) { 836 value = params->pages_addr ? 837 amdgpu_vm_map_gart(params->pages_addr, addr) : 838 addr; 839 amdgpu_gmc_set_pte_pde(params->adev, (void *)(uintptr_t)pe, 840 i, value, flags); 841 addr += incr; 842 } 843 } 844 845 static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm, 846 void *owner) 847 { 848 struct amdgpu_sync sync; 849 int r; 850 851 amdgpu_sync_create(&sync); 852 amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner, false); 853 r = amdgpu_sync_wait(&sync, true); 854 amdgpu_sync_free(&sync); 855 856 return r; 857 } 858 859 /* 860 * amdgpu_vm_update_pde - update a single level in the hierarchy 861 * 862 * @param: parameters for the update 863 * @vm: requested vm 864 * @parent: parent directory 865 * @entry: entry to update 866 * 867 * Makes sure the requested entry in parent is up to date. 868 */ 869 static void amdgpu_vm_update_pde(struct amdgpu_pte_update_params *params, 870 struct amdgpu_vm *vm, 871 struct amdgpu_vm_pt *parent, 872 struct amdgpu_vm_pt *entry) 873 { 874 struct amdgpu_bo *bo = parent->base.bo, *pbo; 875 uint64_t pde, pt, flags; 876 unsigned level; 877 878 /* Don't update huge pages here */ 879 if (entry->huge) 880 return; 881 882 for (level = 0, pbo = bo->parent; pbo; ++level) 883 pbo = pbo->parent; 884 885 level += params->adev->vm_manager.root_level; 886 pt = amdgpu_bo_gpu_offset(entry->base.bo); 887 flags = AMDGPU_PTE_VALID; 888 amdgpu_gmc_get_vm_pde(params->adev, level, &pt, &flags); 889 pde = (entry - parent->entries) * 8; 890 if (bo->shadow) 891 params->func(params, bo->shadow, pde, pt, 1, 0, flags); 892 params->func(params, bo, pde, pt, 1, 0, flags); 893 } 894 895 /* 896 * amdgpu_vm_invalidate_level - mark all PD levels as invalid 897 * 898 * @parent: parent PD 899 * 900 * Mark all PD level as invalid after an error. 901 */ 902 static void amdgpu_vm_invalidate_level(struct amdgpu_device *adev, 903 struct amdgpu_vm *vm, 904 struct amdgpu_vm_pt *parent, 905 unsigned level) 906 { 907 unsigned pt_idx, num_entries; 908 909 /* 910 * Recurse into the subdirectories. This recursion is harmless because 911 * we only have a maximum of 5 layers. 912 */ 913 num_entries = amdgpu_vm_num_entries(adev, level); 914 for (pt_idx = 0; pt_idx < num_entries; ++pt_idx) { 915 struct amdgpu_vm_pt *entry = &parent->entries[pt_idx]; 916 917 if (!entry->base.bo) 918 continue; 919 920 if (!entry->base.moved) 921 list_move(&entry->base.vm_status, &vm->relocated); 922 amdgpu_vm_invalidate_level(adev, vm, entry, level + 1); 923 } 924 } 925 926 /* 927 * amdgpu_vm_update_directories - make sure that all directories are valid 928 * 929 * @adev: amdgpu_device pointer 930 * @vm: requested vm 931 * 932 * Makes sure all directories are up to date. 933 * Returns 0 for success, error for failure. 934 */ 935 int amdgpu_vm_update_directories(struct amdgpu_device *adev, 936 struct amdgpu_vm *vm) 937 { 938 struct amdgpu_pte_update_params params; 939 struct amdgpu_job *job; 940 unsigned ndw = 0; 941 int r = 0; 942 943 if (list_empty(&vm->relocated)) 944 return 0; 945 946 restart: 947 memset(¶ms, 0, sizeof(params)); 948 params.adev = adev; 949 950 if (vm->use_cpu_for_update) { 951 struct amdgpu_vm_bo_base *bo_base; 952 953 list_for_each_entry(bo_base, &vm->relocated, vm_status) { 954 r = amdgpu_bo_kmap(bo_base->bo, NULL); 955 if (unlikely(r)) 956 return r; 957 } 958 959 r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM); 960 if (unlikely(r)) 961 return r; 962 963 params.func = amdgpu_vm_cpu_set_ptes; 964 } else { 965 ndw = 512 * 8; 966 r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job); 967 if (r) 968 return r; 969 970 params.ib = &job->ibs[0]; 971 params.func = amdgpu_vm_do_set_ptes; 972 } 973 974 while (!list_empty(&vm->relocated)) { 975 struct amdgpu_vm_bo_base *bo_base, *parent; 976 struct amdgpu_vm_pt *pt, *entry; 977 struct amdgpu_bo *bo; 978 979 bo_base = list_first_entry(&vm->relocated, 980 struct amdgpu_vm_bo_base, 981 vm_status); 982 bo_base->moved = false; 983 list_move(&bo_base->vm_status, &vm->idle); 984 985 bo = bo_base->bo->parent; 986 if (!bo) 987 continue; 988 989 parent = list_first_entry(&bo->va, struct amdgpu_vm_bo_base, 990 bo_list); 991 pt = container_of(parent, struct amdgpu_vm_pt, base); 992 entry = container_of(bo_base, struct amdgpu_vm_pt, base); 993 994 amdgpu_vm_update_pde(¶ms, vm, pt, entry); 995 996 if (!vm->use_cpu_for_update && 997 (ndw - params.ib->length_dw) < 32) 998 break; 999 } 1000 1001 if (vm->use_cpu_for_update) { 1002 /* Flush HDP */ 1003 mb(); 1004 amdgpu_asic_flush_hdp(adev, NULL); 1005 } else if (params.ib->length_dw == 0) { 1006 amdgpu_job_free(job); 1007 } else { 1008 struct amdgpu_bo *root = vm->root.base.bo; 1009 struct amdgpu_ring *ring; 1010 struct dma_fence *fence; 1011 1012 ring = container_of(vm->entity.sched, struct amdgpu_ring, 1013 sched); 1014 1015 amdgpu_ring_pad_ib(ring, params.ib); 1016 amdgpu_sync_resv(adev, &job->sync, root->tbo.resv, 1017 AMDGPU_FENCE_OWNER_VM, false); 1018 WARN_ON(params.ib->length_dw > ndw); 1019 r = amdgpu_job_submit(job, ring, &vm->entity, 1020 AMDGPU_FENCE_OWNER_VM, &fence); 1021 if (r) 1022 goto error; 1023 1024 amdgpu_bo_fence(root, fence, true); 1025 dma_fence_put(vm->last_update); 1026 vm->last_update = fence; 1027 } 1028 1029 if (!list_empty(&vm->relocated)) 1030 goto restart; 1031 1032 return 0; 1033 1034 error: 1035 amdgpu_vm_invalidate_level(adev, vm, &vm->root, 1036 adev->vm_manager.root_level); 1037 amdgpu_job_free(job); 1038 return r; 1039 } 1040 1041 /** 1042 * amdgpu_vm_find_entry - find the entry for an address 1043 * 1044 * @p: see amdgpu_pte_update_params definition 1045 * @addr: virtual address in question 1046 * @entry: resulting entry or NULL 1047 * @parent: parent entry 1048 * 1049 * Find the vm_pt entry and it's parent for the given address. 1050 */ 1051 void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr, 1052 struct amdgpu_vm_pt **entry, 1053 struct amdgpu_vm_pt **parent) 1054 { 1055 unsigned level = p->adev->vm_manager.root_level; 1056 1057 *parent = NULL; 1058 *entry = &p->vm->root; 1059 while ((*entry)->entries) { 1060 unsigned shift = amdgpu_vm_level_shift(p->adev, level++); 1061 1062 *parent = *entry; 1063 *entry = &(*entry)->entries[addr >> shift]; 1064 addr &= (1ULL << shift) - 1; 1065 } 1066 1067 if (level != AMDGPU_VM_PTB) 1068 *entry = NULL; 1069 } 1070 1071 /** 1072 * amdgpu_vm_handle_huge_pages - handle updating the PD with huge pages 1073 * 1074 * @p: see amdgpu_pte_update_params definition 1075 * @entry: vm_pt entry to check 1076 * @parent: parent entry 1077 * @nptes: number of PTEs updated with this operation 1078 * @dst: destination address where the PTEs should point to 1079 * @flags: access flags fro the PTEs 1080 * 1081 * Check if we can update the PD with a huge page. 1082 */ 1083 static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p, 1084 struct amdgpu_vm_pt *entry, 1085 struct amdgpu_vm_pt *parent, 1086 unsigned nptes, uint64_t dst, 1087 uint64_t flags) 1088 { 1089 uint64_t pde; 1090 1091 /* In the case of a mixed PT the PDE must point to it*/ 1092 if (p->adev->asic_type >= CHIP_VEGA10 && !p->src && 1093 nptes == AMDGPU_VM_PTE_COUNT(p->adev)) { 1094 /* Set the huge page flag to stop scanning at this PDE */ 1095 flags |= AMDGPU_PDE_PTE; 1096 } 1097 1098 if (!(flags & AMDGPU_PDE_PTE)) { 1099 if (entry->huge) { 1100 /* Add the entry to the relocated list to update it. */ 1101 entry->huge = false; 1102 list_move(&entry->base.vm_status, &p->vm->relocated); 1103 } 1104 return; 1105 } 1106 1107 entry->huge = true; 1108 amdgpu_gmc_get_vm_pde(p->adev, AMDGPU_VM_PDB0, &dst, &flags); 1109 1110 pde = (entry - parent->entries) * 8; 1111 if (parent->base.bo->shadow) 1112 p->func(p, parent->base.bo->shadow, pde, dst, 1, 0, flags); 1113 p->func(p, parent->base.bo, pde, dst, 1, 0, flags); 1114 } 1115 1116 /** 1117 * amdgpu_vm_update_ptes - make sure that page tables are valid 1118 * 1119 * @params: see amdgpu_pte_update_params definition 1120 * @vm: requested vm 1121 * @start: start of GPU address range 1122 * @end: end of GPU address range 1123 * @dst: destination address to map to, the next dst inside the function 1124 * @flags: mapping flags 1125 * 1126 * Update the page tables in the range @start - @end. 1127 * Returns 0 for success, -EINVAL for failure. 1128 */ 1129 static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params, 1130 uint64_t start, uint64_t end, 1131 uint64_t dst, uint64_t flags) 1132 { 1133 struct amdgpu_device *adev = params->adev; 1134 const uint64_t mask = AMDGPU_VM_PTE_COUNT(adev) - 1; 1135 1136 uint64_t addr, pe_start; 1137 struct amdgpu_bo *pt; 1138 unsigned nptes; 1139 1140 /* walk over the address space and update the page tables */ 1141 for (addr = start; addr < end; addr += nptes, 1142 dst += nptes * AMDGPU_GPU_PAGE_SIZE) { 1143 struct amdgpu_vm_pt *entry, *parent; 1144 1145 amdgpu_vm_get_entry(params, addr, &entry, &parent); 1146 if (!entry) 1147 return -ENOENT; 1148 1149 if ((addr & ~mask) == (end & ~mask)) 1150 nptes = end - addr; 1151 else 1152 nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask); 1153 1154 amdgpu_vm_handle_huge_pages(params, entry, parent, 1155 nptes, dst, flags); 1156 /* We don't need to update PTEs for huge pages */ 1157 if (entry->huge) 1158 continue; 1159 1160 pt = entry->base.bo; 1161 pe_start = (addr & mask) * 8; 1162 if (pt->shadow) 1163 params->func(params, pt->shadow, pe_start, dst, nptes, 1164 AMDGPU_GPU_PAGE_SIZE, flags); 1165 params->func(params, pt, pe_start, dst, nptes, 1166 AMDGPU_GPU_PAGE_SIZE, flags); 1167 } 1168 1169 return 0; 1170 } 1171 1172 /* 1173 * amdgpu_vm_frag_ptes - add fragment information to PTEs 1174 * 1175 * @params: see amdgpu_pte_update_params definition 1176 * @vm: requested vm 1177 * @start: first PTE to handle 1178 * @end: last PTE to handle 1179 * @dst: addr those PTEs should point to 1180 * @flags: hw mapping flags 1181 * Returns 0 for success, -EINVAL for failure. 1182 */ 1183 static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params *params, 1184 uint64_t start, uint64_t end, 1185 uint64_t dst, uint64_t flags) 1186 { 1187 /** 1188 * The MC L1 TLB supports variable sized pages, based on a fragment 1189 * field in the PTE. When this field is set to a non-zero value, page 1190 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE 1191 * flags are considered valid for all PTEs within the fragment range 1192 * and corresponding mappings are assumed to be physically contiguous. 1193 * 1194 * The L1 TLB can store a single PTE for the whole fragment, 1195 * significantly increasing the space available for translation 1196 * caching. This leads to large improvements in throughput when the 1197 * TLB is under pressure. 1198 * 1199 * The L2 TLB distributes small and large fragments into two 1200 * asymmetric partitions. The large fragment cache is significantly 1201 * larger. Thus, we try to use large fragments wherever possible. 1202 * Userspace can support this by aligning virtual base address and 1203 * allocation size to the fragment size. 1204 */ 1205 unsigned max_frag = params->adev->vm_manager.fragment_size; 1206 int r; 1207 1208 /* system pages are non continuously */ 1209 if (params->src || !(flags & AMDGPU_PTE_VALID)) 1210 return amdgpu_vm_update_ptes(params, start, end, dst, flags); 1211 1212 while (start != end) { 1213 uint64_t frag_flags, frag_end; 1214 unsigned frag; 1215 1216 /* This intentionally wraps around if no bit is set */ 1217 frag = min((unsigned)ffs(start) - 1, 1218 (unsigned)fls64(end - start) - 1); 1219 if (frag >= max_frag) { 1220 frag_flags = AMDGPU_PTE_FRAG(max_frag); 1221 frag_end = end & ~((1ULL << max_frag) - 1); 1222 } else { 1223 frag_flags = AMDGPU_PTE_FRAG(frag); 1224 frag_end = start + (1 << frag); 1225 } 1226 1227 r = amdgpu_vm_update_ptes(params, start, frag_end, dst, 1228 flags | frag_flags); 1229 if (r) 1230 return r; 1231 1232 dst += (frag_end - start) * AMDGPU_GPU_PAGE_SIZE; 1233 start = frag_end; 1234 } 1235 1236 return 0; 1237 } 1238 1239 /** 1240 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table 1241 * 1242 * @adev: amdgpu_device pointer 1243 * @exclusive: fence we need to sync to 1244 * @pages_addr: DMA addresses to use for mapping 1245 * @vm: requested vm 1246 * @start: start of mapped range 1247 * @last: last mapped entry 1248 * @flags: flags for the entries 1249 * @addr: addr to set the area to 1250 * @fence: optional resulting fence 1251 * 1252 * Fill in the page table entries between @start and @last. 1253 * Returns 0 for success, -EINVAL for failure. 1254 */ 1255 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, 1256 struct dma_fence *exclusive, 1257 dma_addr_t *pages_addr, 1258 struct amdgpu_vm *vm, 1259 uint64_t start, uint64_t last, 1260 uint64_t flags, uint64_t addr, 1261 struct dma_fence **fence) 1262 { 1263 struct amdgpu_ring *ring; 1264 void *owner = AMDGPU_FENCE_OWNER_VM; 1265 unsigned nptes, ncmds, ndw; 1266 struct amdgpu_job *job; 1267 struct amdgpu_pte_update_params params; 1268 struct dma_fence *f = NULL; 1269 int r; 1270 1271 memset(¶ms, 0, sizeof(params)); 1272 params.adev = adev; 1273 params.vm = vm; 1274 1275 /* sync to everything on unmapping */ 1276 if (!(flags & AMDGPU_PTE_VALID)) 1277 owner = AMDGPU_FENCE_OWNER_UNDEFINED; 1278 1279 if (vm->use_cpu_for_update) { 1280 /* params.src is used as flag to indicate system Memory */ 1281 if (pages_addr) 1282 params.src = ~0; 1283 1284 /* Wait for PT BOs to be free. PTs share the same resv. object 1285 * as the root PD BO 1286 */ 1287 r = amdgpu_vm_wait_pd(adev, vm, owner); 1288 if (unlikely(r)) 1289 return r; 1290 1291 params.func = amdgpu_vm_cpu_set_ptes; 1292 params.pages_addr = pages_addr; 1293 return amdgpu_vm_frag_ptes(¶ms, start, last + 1, 1294 addr, flags); 1295 } 1296 1297 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched); 1298 1299 nptes = last - start + 1; 1300 1301 /* 1302 * reserve space for two commands every (1 << BLOCK_SIZE) 1303 * entries or 2k dwords (whatever is smaller) 1304 * 1305 * The second command is for the shadow pagetables. 1306 */ 1307 if (vm->root.base.bo->shadow) 1308 ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2; 1309 else 1310 ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1); 1311 1312 /* padding, etc. */ 1313 ndw = 64; 1314 1315 if (pages_addr) { 1316 /* copy commands needed */ 1317 ndw += ncmds * adev->vm_manager.vm_pte_funcs->copy_pte_num_dw; 1318 1319 /* and also PTEs */ 1320 ndw += nptes * 2; 1321 1322 params.func = amdgpu_vm_do_copy_ptes; 1323 1324 } else { 1325 /* set page commands needed */ 1326 ndw += ncmds * 10; 1327 1328 /* extra commands for begin/end fragments */ 1329 ndw += 2 * 10 * adev->vm_manager.fragment_size; 1330 1331 params.func = amdgpu_vm_do_set_ptes; 1332 } 1333 1334 r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job); 1335 if (r) 1336 return r; 1337 1338 params.ib = &job->ibs[0]; 1339 1340 if (pages_addr) { 1341 uint64_t *pte; 1342 unsigned i; 1343 1344 /* Put the PTEs at the end of the IB. */ 1345 i = ndw - nptes * 2; 1346 pte= (uint64_t *)&(job->ibs->ptr[i]); 1347 params.src = job->ibs->gpu_addr + i * 4; 1348 1349 for (i = 0; i < nptes; ++i) { 1350 pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i * 1351 AMDGPU_GPU_PAGE_SIZE); 1352 pte[i] |= flags; 1353 } 1354 addr = 0; 1355 } 1356 1357 r = amdgpu_sync_fence(adev, &job->sync, exclusive, false); 1358 if (r) 1359 goto error_free; 1360 1361 r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv, 1362 owner, false); 1363 if (r) 1364 goto error_free; 1365 1366 r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv); 1367 if (r) 1368 goto error_free; 1369 1370 r = amdgpu_vm_frag_ptes(¶ms, start, last + 1, addr, flags); 1371 if (r) 1372 goto error_free; 1373 1374 amdgpu_ring_pad_ib(ring, params.ib); 1375 WARN_ON(params.ib->length_dw > ndw); 1376 r = amdgpu_job_submit(job, ring, &vm->entity, 1377 AMDGPU_FENCE_OWNER_VM, &f); 1378 if (r) 1379 goto error_free; 1380 1381 amdgpu_bo_fence(vm->root.base.bo, f, true); 1382 dma_fence_put(*fence); 1383 *fence = f; 1384 return 0; 1385 1386 error_free: 1387 amdgpu_job_free(job); 1388 return r; 1389 } 1390 1391 /** 1392 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks 1393 * 1394 * @adev: amdgpu_device pointer 1395 * @exclusive: fence we need to sync to 1396 * @pages_addr: DMA addresses to use for mapping 1397 * @vm: requested vm 1398 * @mapping: mapped range and flags to use for the update 1399 * @flags: HW flags for the mapping 1400 * @nodes: array of drm_mm_nodes with the MC addresses 1401 * @fence: optional resulting fence 1402 * 1403 * Split the mapping into smaller chunks so that each update fits 1404 * into a SDMA IB. 1405 * Returns 0 for success, -EINVAL for failure. 1406 */ 1407 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, 1408 struct dma_fence *exclusive, 1409 dma_addr_t *pages_addr, 1410 struct amdgpu_vm *vm, 1411 struct amdgpu_bo_va_mapping *mapping, 1412 uint64_t flags, 1413 struct drm_mm_node *nodes, 1414 struct dma_fence **fence) 1415 { 1416 unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size; 1417 uint64_t pfn, start = mapping->start; 1418 int r; 1419 1420 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here 1421 * but in case of something, we filter the flags in first place 1422 */ 1423 if (!(mapping->flags & AMDGPU_PTE_READABLE)) 1424 flags &= ~AMDGPU_PTE_READABLE; 1425 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) 1426 flags &= ~AMDGPU_PTE_WRITEABLE; 1427 1428 flags &= ~AMDGPU_PTE_EXECUTABLE; 1429 flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; 1430 1431 flags &= ~AMDGPU_PTE_MTYPE_MASK; 1432 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK); 1433 1434 if ((mapping->flags & AMDGPU_PTE_PRT) && 1435 (adev->asic_type >= CHIP_VEGA10)) { 1436 flags |= AMDGPU_PTE_PRT; 1437 flags &= ~AMDGPU_PTE_VALID; 1438 } 1439 1440 trace_amdgpu_vm_bo_update(mapping); 1441 1442 pfn = mapping->offset >> PAGE_SHIFT; 1443 if (nodes) { 1444 while (pfn >= nodes->size) { 1445 pfn -= nodes->size; 1446 ++nodes; 1447 } 1448 } 1449 1450 do { 1451 dma_addr_t *dma_addr = NULL; 1452 uint64_t max_entries; 1453 uint64_t addr, last; 1454 1455 if (nodes) { 1456 addr = nodes->start << PAGE_SHIFT; 1457 max_entries = (nodes->size - pfn) * 1458 (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE); 1459 } else { 1460 addr = 0; 1461 max_entries = S64_MAX; 1462 } 1463 1464 if (pages_addr) { 1465 uint64_t count; 1466 1467 max_entries = min(max_entries, 16ull * 1024ull); 1468 for (count = 1; 1469 count < max_entries / (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE); 1470 ++count) { 1471 uint64_t idx = pfn + count; 1472 1473 if (pages_addr[idx] != 1474 (pages_addr[idx - 1] + PAGE_SIZE)) 1475 break; 1476 } 1477 1478 if (count < min_linear_pages) { 1479 addr = pfn << PAGE_SHIFT; 1480 dma_addr = pages_addr; 1481 } else { 1482 addr = pages_addr[pfn]; 1483 max_entries = count * (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE); 1484 } 1485 1486 } else if (flags & AMDGPU_PTE_VALID) { 1487 addr += adev->vm_manager.vram_base_offset; 1488 addr += pfn << PAGE_SHIFT; 1489 } 1490 1491 last = min((uint64_t)mapping->last, start + max_entries - 1); 1492 r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm, 1493 start, last, flags, addr, 1494 fence); 1495 if (r) 1496 return r; 1497 1498 pfn += (last - start + 1) / (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE); 1499 if (nodes && nodes->size == pfn) { 1500 pfn = 0; 1501 ++nodes; 1502 } 1503 start = last + 1; 1504 1505 } while (unlikely(start != mapping->last + 1)); 1506 1507 return 0; 1508 } 1509 1510 /** 1511 * amdgpu_vm_bo_update - update all BO mappings in the vm page table 1512 * 1513 * @adev: amdgpu_device pointer 1514 * @bo_va: requested BO and VM object 1515 * @clear: if true clear the entries 1516 * 1517 * Fill in the page table entries for @bo_va. 1518 * Returns 0 for success, -EINVAL for failure. 1519 */ 1520 int amdgpu_vm_bo_update(struct amdgpu_device *adev, 1521 struct amdgpu_bo_va *bo_va, 1522 bool clear) 1523 { 1524 struct amdgpu_bo *bo = bo_va->base.bo; 1525 struct amdgpu_vm *vm = bo_va->base.vm; 1526 struct amdgpu_bo_va_mapping *mapping; 1527 dma_addr_t *pages_addr = NULL; 1528 struct ttm_mem_reg *mem; 1529 struct drm_mm_node *nodes; 1530 struct dma_fence *exclusive, **last_update; 1531 uint64_t flags; 1532 int r; 1533 1534 if (clear || !bo_va->base.bo) { 1535 mem = NULL; 1536 nodes = NULL; 1537 exclusive = NULL; 1538 } else { 1539 struct ttm_dma_tt *ttm; 1540 1541 mem = &bo_va->base.bo->tbo.mem; 1542 nodes = mem->mm_node; 1543 if (mem->mem_type == TTM_PL_TT) { 1544 ttm = container_of(bo_va->base.bo->tbo.ttm, 1545 struct ttm_dma_tt, ttm); 1546 pages_addr = ttm->dma_address; 1547 } 1548 exclusive = reservation_object_get_excl(bo->tbo.resv); 1549 } 1550 1551 if (bo) 1552 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem); 1553 else 1554 flags = 0x0; 1555 1556 if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv)) 1557 last_update = &vm->last_update; 1558 else 1559 last_update = &bo_va->last_pt_update; 1560 1561 if (!clear && bo_va->base.moved) { 1562 bo_va->base.moved = false; 1563 list_splice_init(&bo_va->valids, &bo_va->invalids); 1564 1565 } else if (bo_va->cleared != clear) { 1566 list_splice_init(&bo_va->valids, &bo_va->invalids); 1567 } 1568 1569 list_for_each_entry(mapping, &bo_va->invalids, list) { 1570 r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm, 1571 mapping, flags, nodes, 1572 last_update); 1573 if (r) 1574 return r; 1575 } 1576 1577 if (vm->use_cpu_for_update) { 1578 /* Flush HDP */ 1579 mb(); 1580 amdgpu_asic_flush_hdp(adev, NULL); 1581 } 1582 1583 spin_lock(&vm->moved_lock); 1584 list_del_init(&bo_va->base.vm_status); 1585 spin_unlock(&vm->moved_lock); 1586 1587 /* If the BO is not in its preferred location add it back to 1588 * the evicted list so that it gets validated again on the 1589 * next command submission. 1590 */ 1591 if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) { 1592 uint32_t mem_type = bo->tbo.mem.mem_type; 1593 1594 if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type))) 1595 list_add_tail(&bo_va->base.vm_status, &vm->evicted); 1596 else 1597 list_add(&bo_va->base.vm_status, &vm->idle); 1598 } 1599 1600 list_splice_init(&bo_va->invalids, &bo_va->valids); 1601 bo_va->cleared = clear; 1602 1603 if (trace_amdgpu_vm_bo_mapping_enabled()) { 1604 list_for_each_entry(mapping, &bo_va->valids, list) 1605 trace_amdgpu_vm_bo_mapping(mapping); 1606 } 1607 1608 return 0; 1609 } 1610 1611 /** 1612 * amdgpu_vm_update_prt_state - update the global PRT state 1613 */ 1614 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev) 1615 { 1616 unsigned long flags; 1617 bool enable; 1618 1619 spin_lock_irqsave(&adev->vm_manager.prt_lock, flags); 1620 enable = !!atomic_read(&adev->vm_manager.num_prt_users); 1621 adev->gmc.gmc_funcs->set_prt(adev, enable); 1622 spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags); 1623 } 1624 1625 /** 1626 * amdgpu_vm_prt_get - add a PRT user 1627 */ 1628 static void amdgpu_vm_prt_get(struct amdgpu_device *adev) 1629 { 1630 if (!adev->gmc.gmc_funcs->set_prt) 1631 return; 1632 1633 if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1) 1634 amdgpu_vm_update_prt_state(adev); 1635 } 1636 1637 /** 1638 * amdgpu_vm_prt_put - drop a PRT user 1639 */ 1640 static void amdgpu_vm_prt_put(struct amdgpu_device *adev) 1641 { 1642 if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0) 1643 amdgpu_vm_update_prt_state(adev); 1644 } 1645 1646 /** 1647 * amdgpu_vm_prt_cb - callback for updating the PRT status 1648 */ 1649 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb) 1650 { 1651 struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb); 1652 1653 amdgpu_vm_prt_put(cb->adev); 1654 kfree(cb); 1655 } 1656 1657 /** 1658 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status 1659 */ 1660 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev, 1661 struct dma_fence *fence) 1662 { 1663 struct amdgpu_prt_cb *cb; 1664 1665 if (!adev->gmc.gmc_funcs->set_prt) 1666 return; 1667 1668 cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL); 1669 if (!cb) { 1670 /* Last resort when we are OOM */ 1671 if (fence) 1672 dma_fence_wait(fence, false); 1673 1674 amdgpu_vm_prt_put(adev); 1675 } else { 1676 cb->adev = adev; 1677 if (!fence || dma_fence_add_callback(fence, &cb->cb, 1678 amdgpu_vm_prt_cb)) 1679 amdgpu_vm_prt_cb(fence, &cb->cb); 1680 } 1681 } 1682 1683 /** 1684 * amdgpu_vm_free_mapping - free a mapping 1685 * 1686 * @adev: amdgpu_device pointer 1687 * @vm: requested vm 1688 * @mapping: mapping to be freed 1689 * @fence: fence of the unmap operation 1690 * 1691 * Free a mapping and make sure we decrease the PRT usage count if applicable. 1692 */ 1693 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev, 1694 struct amdgpu_vm *vm, 1695 struct amdgpu_bo_va_mapping *mapping, 1696 struct dma_fence *fence) 1697 { 1698 if (mapping->flags & AMDGPU_PTE_PRT) 1699 amdgpu_vm_add_prt_cb(adev, fence); 1700 kfree(mapping); 1701 } 1702 1703 /** 1704 * amdgpu_vm_prt_fini - finish all prt mappings 1705 * 1706 * @adev: amdgpu_device pointer 1707 * @vm: requested vm 1708 * 1709 * Register a cleanup callback to disable PRT support after VM dies. 1710 */ 1711 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1712 { 1713 struct reservation_object *resv = vm->root.base.bo->tbo.resv; 1714 struct dma_fence *excl, **shared; 1715 unsigned i, shared_count; 1716 int r; 1717 1718 r = reservation_object_get_fences_rcu(resv, &excl, 1719 &shared_count, &shared); 1720 if (r) { 1721 /* Not enough memory to grab the fence list, as last resort 1722 * block for all the fences to complete. 1723 */ 1724 reservation_object_wait_timeout_rcu(resv, true, false, 1725 MAX_SCHEDULE_TIMEOUT); 1726 return; 1727 } 1728 1729 /* Add a callback for each fence in the reservation object */ 1730 amdgpu_vm_prt_get(adev); 1731 amdgpu_vm_add_prt_cb(adev, excl); 1732 1733 for (i = 0; i < shared_count; ++i) { 1734 amdgpu_vm_prt_get(adev); 1735 amdgpu_vm_add_prt_cb(adev, shared[i]); 1736 } 1737 1738 kfree(shared); 1739 } 1740 1741 /** 1742 * amdgpu_vm_clear_freed - clear freed BOs in the PT 1743 * 1744 * @adev: amdgpu_device pointer 1745 * @vm: requested vm 1746 * @fence: optional resulting fence (unchanged if no work needed to be done 1747 * or if an error occurred) 1748 * 1749 * Make sure all freed BOs are cleared in the PT. 1750 * Returns 0 for success. 1751 * 1752 * PTs have to be reserved and mutex must be locked! 1753 */ 1754 int amdgpu_vm_clear_freed(struct amdgpu_device *adev, 1755 struct amdgpu_vm *vm, 1756 struct dma_fence **fence) 1757 { 1758 struct amdgpu_bo_va_mapping *mapping; 1759 uint64_t init_pte_value = 0; 1760 struct dma_fence *f = NULL; 1761 int r; 1762 1763 while (!list_empty(&vm->freed)) { 1764 mapping = list_first_entry(&vm->freed, 1765 struct amdgpu_bo_va_mapping, list); 1766 list_del(&mapping->list); 1767 1768 if (vm->pte_support_ats && mapping->start < AMDGPU_VA_HOLE_START) 1769 init_pte_value = AMDGPU_PTE_DEFAULT_ATC; 1770 1771 r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm, 1772 mapping->start, mapping->last, 1773 init_pte_value, 0, &f); 1774 amdgpu_vm_free_mapping(adev, vm, mapping, f); 1775 if (r) { 1776 dma_fence_put(f); 1777 return r; 1778 } 1779 } 1780 1781 if (fence && f) { 1782 dma_fence_put(*fence); 1783 *fence = f; 1784 } else { 1785 dma_fence_put(f); 1786 } 1787 1788 return 0; 1789 1790 } 1791 1792 /** 1793 * amdgpu_vm_handle_moved - handle moved BOs in the PT 1794 * 1795 * @adev: amdgpu_device pointer 1796 * @vm: requested vm 1797 * @sync: sync object to add fences to 1798 * 1799 * Make sure all BOs which are moved are updated in the PTs. 1800 * Returns 0 for success. 1801 * 1802 * PTs have to be reserved! 1803 */ 1804 int amdgpu_vm_handle_moved(struct amdgpu_device *adev, 1805 struct amdgpu_vm *vm) 1806 { 1807 struct amdgpu_bo_va *bo_va, *tmp; 1808 struct list_head moved; 1809 bool clear; 1810 int r; 1811 1812 INIT_LIST_HEAD(&moved); 1813 spin_lock(&vm->moved_lock); 1814 list_splice_init(&vm->moved, &moved); 1815 spin_unlock(&vm->moved_lock); 1816 1817 list_for_each_entry_safe(bo_va, tmp, &moved, base.vm_status) { 1818 struct reservation_object *resv = bo_va->base.bo->tbo.resv; 1819 1820 /* Per VM BOs never need to bo cleared in the page tables */ 1821 if (resv == vm->root.base.bo->tbo.resv) 1822 clear = false; 1823 /* Try to reserve the BO to avoid clearing its ptes */ 1824 else if (!amdgpu_vm_debug && reservation_object_trylock(resv)) 1825 clear = false; 1826 /* Somebody else is using the BO right now */ 1827 else 1828 clear = true; 1829 1830 r = amdgpu_vm_bo_update(adev, bo_va, clear); 1831 if (r) { 1832 spin_lock(&vm->moved_lock); 1833 list_splice(&moved, &vm->moved); 1834 spin_unlock(&vm->moved_lock); 1835 return r; 1836 } 1837 1838 if (!clear && resv != vm->root.base.bo->tbo.resv) 1839 reservation_object_unlock(resv); 1840 1841 } 1842 1843 return 0; 1844 } 1845 1846 /** 1847 * amdgpu_vm_bo_add - add a bo to a specific vm 1848 * 1849 * @adev: amdgpu_device pointer 1850 * @vm: requested vm 1851 * @bo: amdgpu buffer object 1852 * 1853 * Add @bo into the requested vm. 1854 * Add @bo to the list of bos associated with the vm 1855 * Returns newly added bo_va or NULL for failure 1856 * 1857 * Object has to be reserved! 1858 */ 1859 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, 1860 struct amdgpu_vm *vm, 1861 struct amdgpu_bo *bo) 1862 { 1863 struct amdgpu_bo_va *bo_va; 1864 1865 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL); 1866 if (bo_va == NULL) { 1867 return NULL; 1868 } 1869 amdgpu_vm_bo_base_init(&bo_va->base, vm, bo); 1870 1871 bo_va->ref_count = 1; 1872 INIT_LIST_HEAD(&bo_va->valids); 1873 INIT_LIST_HEAD(&bo_va->invalids); 1874 1875 return bo_va; 1876 } 1877 1878 1879 /** 1880 * amdgpu_vm_bo_insert_mapping - insert a new mapping 1881 * 1882 * @adev: amdgpu_device pointer 1883 * @bo_va: bo_va to store the address 1884 * @mapping: the mapping to insert 1885 * 1886 * Insert a new mapping into all structures. 1887 */ 1888 static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev, 1889 struct amdgpu_bo_va *bo_va, 1890 struct amdgpu_bo_va_mapping *mapping) 1891 { 1892 struct amdgpu_vm *vm = bo_va->base.vm; 1893 struct amdgpu_bo *bo = bo_va->base.bo; 1894 1895 mapping->bo_va = bo_va; 1896 list_add(&mapping->list, &bo_va->invalids); 1897 amdgpu_vm_it_insert(mapping, &vm->va); 1898 1899 if (mapping->flags & AMDGPU_PTE_PRT) 1900 amdgpu_vm_prt_get(adev); 1901 1902 if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv && 1903 !bo_va->base.moved) { 1904 spin_lock(&vm->moved_lock); 1905 list_move(&bo_va->base.vm_status, &vm->moved); 1906 spin_unlock(&vm->moved_lock); 1907 } 1908 trace_amdgpu_vm_bo_map(bo_va, mapping); 1909 } 1910 1911 /** 1912 * amdgpu_vm_bo_map - map bo inside a vm 1913 * 1914 * @adev: amdgpu_device pointer 1915 * @bo_va: bo_va to store the address 1916 * @saddr: where to map the BO 1917 * @offset: requested offset in the BO 1918 * @flags: attributes of pages (read/write/valid/etc.) 1919 * 1920 * Add a mapping of the BO at the specefied addr into the VM. 1921 * Returns 0 for success, error for failure. 1922 * 1923 * Object has to be reserved and unreserved outside! 1924 */ 1925 int amdgpu_vm_bo_map(struct amdgpu_device *adev, 1926 struct amdgpu_bo_va *bo_va, 1927 uint64_t saddr, uint64_t offset, 1928 uint64_t size, uint64_t flags) 1929 { 1930 struct amdgpu_bo_va_mapping *mapping, *tmp; 1931 struct amdgpu_bo *bo = bo_va->base.bo; 1932 struct amdgpu_vm *vm = bo_va->base.vm; 1933 uint64_t eaddr; 1934 1935 /* validate the parameters */ 1936 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || 1937 size == 0 || size & AMDGPU_GPU_PAGE_MASK) 1938 return -EINVAL; 1939 1940 /* make sure object fit at this offset */ 1941 eaddr = saddr + size - 1; 1942 if (saddr >= eaddr || 1943 (bo && offset + size > amdgpu_bo_size(bo))) 1944 return -EINVAL; 1945 1946 saddr /= AMDGPU_GPU_PAGE_SIZE; 1947 eaddr /= AMDGPU_GPU_PAGE_SIZE; 1948 1949 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); 1950 if (tmp) { 1951 /* bo and tmp overlap, invalid addr */ 1952 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with " 1953 "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr, 1954 tmp->start, tmp->last + 1); 1955 return -EINVAL; 1956 } 1957 1958 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 1959 if (!mapping) 1960 return -ENOMEM; 1961 1962 mapping->start = saddr; 1963 mapping->last = eaddr; 1964 mapping->offset = offset; 1965 mapping->flags = flags; 1966 1967 amdgpu_vm_bo_insert_map(adev, bo_va, mapping); 1968 1969 return 0; 1970 } 1971 1972 /** 1973 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings 1974 * 1975 * @adev: amdgpu_device pointer 1976 * @bo_va: bo_va to store the address 1977 * @saddr: where to map the BO 1978 * @offset: requested offset in the BO 1979 * @flags: attributes of pages (read/write/valid/etc.) 1980 * 1981 * Add a mapping of the BO at the specefied addr into the VM. Replace existing 1982 * mappings as we do so. 1983 * Returns 0 for success, error for failure. 1984 * 1985 * Object has to be reserved and unreserved outside! 1986 */ 1987 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev, 1988 struct amdgpu_bo_va *bo_va, 1989 uint64_t saddr, uint64_t offset, 1990 uint64_t size, uint64_t flags) 1991 { 1992 struct amdgpu_bo_va_mapping *mapping; 1993 struct amdgpu_bo *bo = bo_va->base.bo; 1994 uint64_t eaddr; 1995 int r; 1996 1997 /* validate the parameters */ 1998 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || 1999 size == 0 || size & AMDGPU_GPU_PAGE_MASK) 2000 return -EINVAL; 2001 2002 /* make sure object fit at this offset */ 2003 eaddr = saddr + size - 1; 2004 if (saddr >= eaddr || 2005 (bo && offset + size > amdgpu_bo_size(bo))) 2006 return -EINVAL; 2007 2008 /* Allocate all the needed memory */ 2009 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 2010 if (!mapping) 2011 return -ENOMEM; 2012 2013 r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size); 2014 if (r) { 2015 kfree(mapping); 2016 return r; 2017 } 2018 2019 saddr /= AMDGPU_GPU_PAGE_SIZE; 2020 eaddr /= AMDGPU_GPU_PAGE_SIZE; 2021 2022 mapping->start = saddr; 2023 mapping->last = eaddr; 2024 mapping->offset = offset; 2025 mapping->flags = flags; 2026 2027 amdgpu_vm_bo_insert_map(adev, bo_va, mapping); 2028 2029 return 0; 2030 } 2031 2032 /** 2033 * amdgpu_vm_bo_unmap - remove bo mapping from vm 2034 * 2035 * @adev: amdgpu_device pointer 2036 * @bo_va: bo_va to remove the address from 2037 * @saddr: where to the BO is mapped 2038 * 2039 * Remove a mapping of the BO at the specefied addr from the VM. 2040 * Returns 0 for success, error for failure. 2041 * 2042 * Object has to be reserved and unreserved outside! 2043 */ 2044 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, 2045 struct amdgpu_bo_va *bo_va, 2046 uint64_t saddr) 2047 { 2048 struct amdgpu_bo_va_mapping *mapping; 2049 struct amdgpu_vm *vm = bo_va->base.vm; 2050 bool valid = true; 2051 2052 saddr /= AMDGPU_GPU_PAGE_SIZE; 2053 2054 list_for_each_entry(mapping, &bo_va->valids, list) { 2055 if (mapping->start == saddr) 2056 break; 2057 } 2058 2059 if (&mapping->list == &bo_va->valids) { 2060 valid = false; 2061 2062 list_for_each_entry(mapping, &bo_va->invalids, list) { 2063 if (mapping->start == saddr) 2064 break; 2065 } 2066 2067 if (&mapping->list == &bo_va->invalids) 2068 return -ENOENT; 2069 } 2070 2071 list_del(&mapping->list); 2072 amdgpu_vm_it_remove(mapping, &vm->va); 2073 mapping->bo_va = NULL; 2074 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 2075 2076 if (valid) 2077 list_add(&mapping->list, &vm->freed); 2078 else 2079 amdgpu_vm_free_mapping(adev, vm, mapping, 2080 bo_va->last_pt_update); 2081 2082 return 0; 2083 } 2084 2085 /** 2086 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range 2087 * 2088 * @adev: amdgpu_device pointer 2089 * @vm: VM structure to use 2090 * @saddr: start of the range 2091 * @size: size of the range 2092 * 2093 * Remove all mappings in a range, split them as appropriate. 2094 * Returns 0 for success, error for failure. 2095 */ 2096 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, 2097 struct amdgpu_vm *vm, 2098 uint64_t saddr, uint64_t size) 2099 { 2100 struct amdgpu_bo_va_mapping *before, *after, *tmp, *next; 2101 LIST_HEAD(removed); 2102 uint64_t eaddr; 2103 2104 eaddr = saddr + size - 1; 2105 saddr /= AMDGPU_GPU_PAGE_SIZE; 2106 eaddr /= AMDGPU_GPU_PAGE_SIZE; 2107 2108 /* Allocate all the needed memory */ 2109 before = kzalloc(sizeof(*before), GFP_KERNEL); 2110 if (!before) 2111 return -ENOMEM; 2112 INIT_LIST_HEAD(&before->list); 2113 2114 after = kzalloc(sizeof(*after), GFP_KERNEL); 2115 if (!after) { 2116 kfree(before); 2117 return -ENOMEM; 2118 } 2119 INIT_LIST_HEAD(&after->list); 2120 2121 /* Now gather all removed mappings */ 2122 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); 2123 while (tmp) { 2124 /* Remember mapping split at the start */ 2125 if (tmp->start < saddr) { 2126 before->start = tmp->start; 2127 before->last = saddr - 1; 2128 before->offset = tmp->offset; 2129 before->flags = tmp->flags; 2130 before->bo_va = tmp->bo_va; 2131 list_add(&before->list, &tmp->bo_va->invalids); 2132 } 2133 2134 /* Remember mapping split at the end */ 2135 if (tmp->last > eaddr) { 2136 after->start = eaddr + 1; 2137 after->last = tmp->last; 2138 after->offset = tmp->offset; 2139 after->offset += after->start - tmp->start; 2140 after->flags = tmp->flags; 2141 after->bo_va = tmp->bo_va; 2142 list_add(&after->list, &tmp->bo_va->invalids); 2143 } 2144 2145 list_del(&tmp->list); 2146 list_add(&tmp->list, &removed); 2147 2148 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr); 2149 } 2150 2151 /* And free them up */ 2152 list_for_each_entry_safe(tmp, next, &removed, list) { 2153 amdgpu_vm_it_remove(tmp, &vm->va); 2154 list_del(&tmp->list); 2155 2156 if (tmp->start < saddr) 2157 tmp->start = saddr; 2158 if (tmp->last > eaddr) 2159 tmp->last = eaddr; 2160 2161 tmp->bo_va = NULL; 2162 list_add(&tmp->list, &vm->freed); 2163 trace_amdgpu_vm_bo_unmap(NULL, tmp); 2164 } 2165 2166 /* Insert partial mapping before the range */ 2167 if (!list_empty(&before->list)) { 2168 amdgpu_vm_it_insert(before, &vm->va); 2169 if (before->flags & AMDGPU_PTE_PRT) 2170 amdgpu_vm_prt_get(adev); 2171 } else { 2172 kfree(before); 2173 } 2174 2175 /* Insert partial mapping after the range */ 2176 if (!list_empty(&after->list)) { 2177 amdgpu_vm_it_insert(after, &vm->va); 2178 if (after->flags & AMDGPU_PTE_PRT) 2179 amdgpu_vm_prt_get(adev); 2180 } else { 2181 kfree(after); 2182 } 2183 2184 return 0; 2185 } 2186 2187 /** 2188 * amdgpu_vm_bo_lookup_mapping - find mapping by address 2189 * 2190 * @vm: the requested VM 2191 * 2192 * Find a mapping by it's address. 2193 */ 2194 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm, 2195 uint64_t addr) 2196 { 2197 return amdgpu_vm_it_iter_first(&vm->va, addr, addr); 2198 } 2199 2200 /** 2201 * amdgpu_vm_bo_rmv - remove a bo to a specific vm 2202 * 2203 * @adev: amdgpu_device pointer 2204 * @bo_va: requested bo_va 2205 * 2206 * Remove @bo_va->bo from the requested vm. 2207 * 2208 * Object have to be reserved! 2209 */ 2210 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, 2211 struct amdgpu_bo_va *bo_va) 2212 { 2213 struct amdgpu_bo_va_mapping *mapping, *next; 2214 struct amdgpu_vm *vm = bo_va->base.vm; 2215 2216 list_del(&bo_va->base.bo_list); 2217 2218 spin_lock(&vm->moved_lock); 2219 list_del(&bo_va->base.vm_status); 2220 spin_unlock(&vm->moved_lock); 2221 2222 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 2223 list_del(&mapping->list); 2224 amdgpu_vm_it_remove(mapping, &vm->va); 2225 mapping->bo_va = NULL; 2226 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 2227 list_add(&mapping->list, &vm->freed); 2228 } 2229 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { 2230 list_del(&mapping->list); 2231 amdgpu_vm_it_remove(mapping, &vm->va); 2232 amdgpu_vm_free_mapping(adev, vm, mapping, 2233 bo_va->last_pt_update); 2234 } 2235 2236 dma_fence_put(bo_va->last_pt_update); 2237 kfree(bo_va); 2238 } 2239 2240 /** 2241 * amdgpu_vm_bo_invalidate - mark the bo as invalid 2242 * 2243 * @adev: amdgpu_device pointer 2244 * @vm: requested vm 2245 * @bo: amdgpu buffer object 2246 * 2247 * Mark @bo as invalid. 2248 */ 2249 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, 2250 struct amdgpu_bo *bo, bool evicted) 2251 { 2252 struct amdgpu_vm_bo_base *bo_base; 2253 2254 /* shadow bo doesn't have bo base, its validation needs its parent */ 2255 if (bo->parent && bo->parent->shadow == bo) 2256 bo = bo->parent; 2257 2258 list_for_each_entry(bo_base, &bo->va, bo_list) { 2259 struct amdgpu_vm *vm = bo_base->vm; 2260 bool was_moved = bo_base->moved; 2261 2262 bo_base->moved = true; 2263 if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) { 2264 if (bo->tbo.type == ttm_bo_type_kernel) 2265 list_move(&bo_base->vm_status, &vm->evicted); 2266 else 2267 list_move_tail(&bo_base->vm_status, 2268 &vm->evicted); 2269 continue; 2270 } 2271 2272 if (was_moved) 2273 continue; 2274 2275 if (bo->tbo.type == ttm_bo_type_kernel) { 2276 list_move(&bo_base->vm_status, &vm->relocated); 2277 } else { 2278 spin_lock(&bo_base->vm->moved_lock); 2279 list_move(&bo_base->vm_status, &vm->moved); 2280 spin_unlock(&bo_base->vm->moved_lock); 2281 } 2282 } 2283 } 2284 2285 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size) 2286 { 2287 /* Total bits covered by PD + PTs */ 2288 unsigned bits = ilog2(vm_size) + 18; 2289 2290 /* Make sure the PD is 4K in size up to 8GB address space. 2291 Above that split equal between PD and PTs */ 2292 if (vm_size <= 8) 2293 return (bits - 9); 2294 else 2295 return ((bits + 3) / 2); 2296 } 2297 2298 /** 2299 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size 2300 * 2301 * @adev: amdgpu_device pointer 2302 * @vm_size: the default vm size if it's set auto 2303 */ 2304 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t vm_size, 2305 uint32_t fragment_size_default, unsigned max_level, 2306 unsigned max_bits) 2307 { 2308 uint64_t tmp; 2309 2310 /* adjust vm size first */ 2311 if (amdgpu_vm_size != -1) { 2312 unsigned max_size = 1 << (max_bits - 30); 2313 2314 vm_size = amdgpu_vm_size; 2315 if (vm_size > max_size) { 2316 dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n", 2317 amdgpu_vm_size, max_size); 2318 vm_size = max_size; 2319 } 2320 } 2321 2322 adev->vm_manager.max_pfn = (uint64_t)vm_size << 18; 2323 2324 tmp = roundup_pow_of_two(adev->vm_manager.max_pfn); 2325 if (amdgpu_vm_block_size != -1) 2326 tmp >>= amdgpu_vm_block_size - 9; 2327 tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1; 2328 adev->vm_manager.num_level = min(max_level, (unsigned)tmp); 2329 switch (adev->vm_manager.num_level) { 2330 case 3: 2331 adev->vm_manager.root_level = AMDGPU_VM_PDB2; 2332 break; 2333 case 2: 2334 adev->vm_manager.root_level = AMDGPU_VM_PDB1; 2335 break; 2336 case 1: 2337 adev->vm_manager.root_level = AMDGPU_VM_PDB0; 2338 break; 2339 default: 2340 dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n"); 2341 } 2342 /* block size depends on vm size and hw setup*/ 2343 if (amdgpu_vm_block_size != -1) 2344 adev->vm_manager.block_size = 2345 min((unsigned)amdgpu_vm_block_size, max_bits 2346 - AMDGPU_GPU_PAGE_SHIFT 2347 - 9 * adev->vm_manager.num_level); 2348 else if (adev->vm_manager.num_level > 1) 2349 adev->vm_manager.block_size = 9; 2350 else 2351 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp); 2352 2353 if (amdgpu_vm_fragment_size == -1) 2354 adev->vm_manager.fragment_size = fragment_size_default; 2355 else 2356 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size; 2357 2358 DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n", 2359 vm_size, adev->vm_manager.num_level + 1, 2360 adev->vm_manager.block_size, 2361 adev->vm_manager.fragment_size); 2362 } 2363 2364 /** 2365 * amdgpu_vm_init - initialize a vm instance 2366 * 2367 * @adev: amdgpu_device pointer 2368 * @vm: requested vm 2369 * @vm_context: Indicates if it GFX or Compute context 2370 * 2371 * Init @vm fields. 2372 */ 2373 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, 2374 int vm_context, unsigned int pasid) 2375 { 2376 struct amdgpu_bo_param bp; 2377 struct amdgpu_bo *root; 2378 const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE, 2379 AMDGPU_VM_PTE_COUNT(adev) * 8); 2380 unsigned ring_instance; 2381 struct amdgpu_ring *ring; 2382 struct drm_sched_rq *rq; 2383 unsigned long size; 2384 uint64_t flags; 2385 int r, i; 2386 2387 vm->va = RB_ROOT_CACHED; 2388 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) 2389 vm->reserved_vmid[i] = NULL; 2390 INIT_LIST_HEAD(&vm->evicted); 2391 INIT_LIST_HEAD(&vm->relocated); 2392 spin_lock_init(&vm->moved_lock); 2393 INIT_LIST_HEAD(&vm->moved); 2394 INIT_LIST_HEAD(&vm->idle); 2395 INIT_LIST_HEAD(&vm->freed); 2396 2397 /* create scheduler entity for page table updates */ 2398 2399 ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring); 2400 ring_instance %= adev->vm_manager.vm_pte_num_rings; 2401 ring = adev->vm_manager.vm_pte_rings[ring_instance]; 2402 rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL]; 2403 r = drm_sched_entity_init(&ring->sched, &vm->entity, 2404 rq, NULL); 2405 if (r) 2406 return r; 2407 2408 vm->pte_support_ats = false; 2409 2410 if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) { 2411 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2412 AMDGPU_VM_USE_CPU_FOR_COMPUTE); 2413 2414 if (adev->asic_type == CHIP_RAVEN) 2415 vm->pte_support_ats = true; 2416 } else { 2417 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2418 AMDGPU_VM_USE_CPU_FOR_GFX); 2419 } 2420 DRM_DEBUG_DRIVER("VM update mode is %s\n", 2421 vm->use_cpu_for_update ? "CPU" : "SDMA"); 2422 WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)), 2423 "CPU update of VM recommended only for large BAR system\n"); 2424 vm->last_update = NULL; 2425 2426 flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 2427 if (vm->use_cpu_for_update) 2428 flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 2429 else 2430 flags |= AMDGPU_GEM_CREATE_SHADOW; 2431 2432 size = amdgpu_vm_bo_size(adev, adev->vm_manager.root_level); 2433 memset(&bp, 0, sizeof(bp)); 2434 bp.size = size; 2435 bp.byte_align = align; 2436 bp.domain = AMDGPU_GEM_DOMAIN_VRAM; 2437 bp.flags = flags; 2438 bp.type = ttm_bo_type_kernel; 2439 bp.resv = NULL; 2440 r = amdgpu_bo_create(adev, &bp, &root); 2441 if (r) 2442 goto error_free_sched_entity; 2443 2444 r = amdgpu_bo_reserve(root, true); 2445 if (r) 2446 goto error_free_root; 2447 2448 r = amdgpu_vm_clear_bo(adev, vm, root, 2449 adev->vm_manager.root_level, 2450 vm->pte_support_ats); 2451 if (r) 2452 goto error_unreserve; 2453 2454 amdgpu_vm_bo_base_init(&vm->root.base, vm, root); 2455 amdgpu_bo_unreserve(vm->root.base.bo); 2456 2457 if (pasid) { 2458 unsigned long flags; 2459 2460 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2461 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1, 2462 GFP_ATOMIC); 2463 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2464 if (r < 0) 2465 goto error_free_root; 2466 2467 vm->pasid = pasid; 2468 } 2469 2470 INIT_KFIFO(vm->faults); 2471 vm->fault_credit = 16; 2472 2473 return 0; 2474 2475 error_unreserve: 2476 amdgpu_bo_unreserve(vm->root.base.bo); 2477 2478 error_free_root: 2479 amdgpu_bo_unref(&vm->root.base.bo->shadow); 2480 amdgpu_bo_unref(&vm->root.base.bo); 2481 vm->root.base.bo = NULL; 2482 2483 error_free_sched_entity: 2484 drm_sched_entity_fini(&ring->sched, &vm->entity); 2485 2486 return r; 2487 } 2488 2489 /** 2490 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM 2491 * 2492 * This only works on GFX VMs that don't have any BOs added and no 2493 * page tables allocated yet. 2494 * 2495 * Changes the following VM parameters: 2496 * - use_cpu_for_update 2497 * - pte_supports_ats 2498 * - pasid (old PASID is released, because compute manages its own PASIDs) 2499 * 2500 * Reinitializes the page directory to reflect the changed ATS 2501 * setting. May leave behind an unused shadow BO for the page 2502 * directory when switching from SDMA updates to CPU updates. 2503 * 2504 * Returns 0 for success, -errno for errors. 2505 */ 2506 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) 2507 { 2508 bool pte_support_ats = (adev->asic_type == CHIP_RAVEN); 2509 int r; 2510 2511 r = amdgpu_bo_reserve(vm->root.base.bo, true); 2512 if (r) 2513 return r; 2514 2515 /* Sanity checks */ 2516 if (!RB_EMPTY_ROOT(&vm->va.rb_root) || vm->root.entries) { 2517 r = -EINVAL; 2518 goto error; 2519 } 2520 2521 /* Check if PD needs to be reinitialized and do it before 2522 * changing any other state, in case it fails. 2523 */ 2524 if (pte_support_ats != vm->pte_support_ats) { 2525 r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo, 2526 adev->vm_manager.root_level, 2527 pte_support_ats); 2528 if (r) 2529 goto error; 2530 } 2531 2532 /* Update VM state */ 2533 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2534 AMDGPU_VM_USE_CPU_FOR_COMPUTE); 2535 vm->pte_support_ats = pte_support_ats; 2536 DRM_DEBUG_DRIVER("VM update mode is %s\n", 2537 vm->use_cpu_for_update ? "CPU" : "SDMA"); 2538 WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)), 2539 "CPU update of VM recommended only for large BAR system\n"); 2540 2541 if (vm->pasid) { 2542 unsigned long flags; 2543 2544 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2545 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); 2546 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2547 2548 vm->pasid = 0; 2549 } 2550 2551 error: 2552 amdgpu_bo_unreserve(vm->root.base.bo); 2553 return r; 2554 } 2555 2556 /** 2557 * amdgpu_vm_free_levels - free PD/PT levels 2558 * 2559 * @adev: amdgpu device structure 2560 * @parent: PD/PT starting level to free 2561 * @level: level of parent structure 2562 * 2563 * Free the page directory or page table level and all sub levels. 2564 */ 2565 static void amdgpu_vm_free_levels(struct amdgpu_device *adev, 2566 struct amdgpu_vm_pt *parent, 2567 unsigned level) 2568 { 2569 unsigned i, num_entries = amdgpu_vm_num_entries(adev, level); 2570 2571 if (parent->base.bo) { 2572 list_del(&parent->base.bo_list); 2573 list_del(&parent->base.vm_status); 2574 amdgpu_bo_unref(&parent->base.bo->shadow); 2575 amdgpu_bo_unref(&parent->base.bo); 2576 } 2577 2578 if (parent->entries) 2579 for (i = 0; i < num_entries; i++) 2580 amdgpu_vm_free_levels(adev, &parent->entries[i], 2581 level + 1); 2582 2583 kvfree(parent->entries); 2584 } 2585 2586 /** 2587 * amdgpu_vm_fini - tear down a vm instance 2588 * 2589 * @adev: amdgpu_device pointer 2590 * @vm: requested vm 2591 * 2592 * Tear down @vm. 2593 * Unbind the VM and remove all bos from the vm bo list 2594 */ 2595 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 2596 { 2597 struct amdgpu_bo_va_mapping *mapping, *tmp; 2598 bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt; 2599 struct amdgpu_bo *root; 2600 u64 fault; 2601 int i, r; 2602 2603 amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm); 2604 2605 /* Clear pending page faults from IH when the VM is destroyed */ 2606 while (kfifo_get(&vm->faults, &fault)) 2607 amdgpu_ih_clear_fault(adev, fault); 2608 2609 if (vm->pasid) { 2610 unsigned long flags; 2611 2612 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2613 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); 2614 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2615 } 2616 2617 drm_sched_entity_fini(vm->entity.sched, &vm->entity); 2618 2619 if (!RB_EMPTY_ROOT(&vm->va.rb_root)) { 2620 dev_err(adev->dev, "still active bo inside vm\n"); 2621 } 2622 rbtree_postorder_for_each_entry_safe(mapping, tmp, 2623 &vm->va.rb_root, rb) { 2624 list_del(&mapping->list); 2625 amdgpu_vm_it_remove(mapping, &vm->va); 2626 kfree(mapping); 2627 } 2628 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { 2629 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) { 2630 amdgpu_vm_prt_fini(adev, vm); 2631 prt_fini_needed = false; 2632 } 2633 2634 list_del(&mapping->list); 2635 amdgpu_vm_free_mapping(adev, vm, mapping, NULL); 2636 } 2637 2638 root = amdgpu_bo_ref(vm->root.base.bo); 2639 r = amdgpu_bo_reserve(root, true); 2640 if (r) { 2641 dev_err(adev->dev, "Leaking page tables because BO reservation failed\n"); 2642 } else { 2643 amdgpu_vm_free_levels(adev, &vm->root, 2644 adev->vm_manager.root_level); 2645 amdgpu_bo_unreserve(root); 2646 } 2647 amdgpu_bo_unref(&root); 2648 dma_fence_put(vm->last_update); 2649 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) 2650 amdgpu_vmid_free_reserved(adev, vm, i); 2651 } 2652 2653 /** 2654 * amdgpu_vm_pasid_fault_credit - Check fault credit for given PASID 2655 * 2656 * @adev: amdgpu_device pointer 2657 * @pasid: PASID do identify the VM 2658 * 2659 * This function is expected to be called in interrupt context. Returns 2660 * true if there was fault credit, false otherwise 2661 */ 2662 bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev, 2663 unsigned int pasid) 2664 { 2665 struct amdgpu_vm *vm; 2666 2667 spin_lock(&adev->vm_manager.pasid_lock); 2668 vm = idr_find(&adev->vm_manager.pasid_idr, pasid); 2669 if (!vm) { 2670 /* VM not found, can't track fault credit */ 2671 spin_unlock(&adev->vm_manager.pasid_lock); 2672 return true; 2673 } 2674 2675 /* No lock needed. only accessed by IRQ handler */ 2676 if (!vm->fault_credit) { 2677 /* Too many faults in this VM */ 2678 spin_unlock(&adev->vm_manager.pasid_lock); 2679 return false; 2680 } 2681 2682 vm->fault_credit--; 2683 spin_unlock(&adev->vm_manager.pasid_lock); 2684 return true; 2685 } 2686 2687 /** 2688 * amdgpu_vm_manager_init - init the VM manager 2689 * 2690 * @adev: amdgpu_device pointer 2691 * 2692 * Initialize the VM manager structures 2693 */ 2694 void amdgpu_vm_manager_init(struct amdgpu_device *adev) 2695 { 2696 unsigned i; 2697 2698 amdgpu_vmid_mgr_init(adev); 2699 2700 adev->vm_manager.fence_context = 2701 dma_fence_context_alloc(AMDGPU_MAX_RINGS); 2702 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) 2703 adev->vm_manager.seqno[i] = 0; 2704 2705 atomic_set(&adev->vm_manager.vm_pte_next_ring, 0); 2706 spin_lock_init(&adev->vm_manager.prt_lock); 2707 atomic_set(&adev->vm_manager.num_prt_users, 0); 2708 2709 /* If not overridden by the user, by default, only in large BAR systems 2710 * Compute VM tables will be updated by CPU 2711 */ 2712 #ifdef CONFIG_X86_64 2713 if (amdgpu_vm_update_mode == -1) { 2714 if (amdgpu_vm_is_large_bar(adev)) 2715 adev->vm_manager.vm_update_mode = 2716 AMDGPU_VM_USE_CPU_FOR_COMPUTE; 2717 else 2718 adev->vm_manager.vm_update_mode = 0; 2719 } else 2720 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode; 2721 #else 2722 adev->vm_manager.vm_update_mode = 0; 2723 #endif 2724 2725 idr_init(&adev->vm_manager.pasid_idr); 2726 spin_lock_init(&adev->vm_manager.pasid_lock); 2727 } 2728 2729 /** 2730 * amdgpu_vm_manager_fini - cleanup VM manager 2731 * 2732 * @adev: amdgpu_device pointer 2733 * 2734 * Cleanup the VM manager and free resources. 2735 */ 2736 void amdgpu_vm_manager_fini(struct amdgpu_device *adev) 2737 { 2738 WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr)); 2739 idr_destroy(&adev->vm_manager.pasid_idr); 2740 2741 amdgpu_vmid_mgr_fini(adev); 2742 } 2743 2744 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 2745 { 2746 union drm_amdgpu_vm *args = data; 2747 struct amdgpu_device *adev = dev->dev_private; 2748 struct amdgpu_fpriv *fpriv = filp->driver_priv; 2749 int r; 2750 2751 switch (args->in.op) { 2752 case AMDGPU_VM_OP_RESERVE_VMID: 2753 /* current, we only have requirement to reserve vmid from gfxhub */ 2754 r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB); 2755 if (r) 2756 return r; 2757 break; 2758 case AMDGPU_VM_OP_UNRESERVE_VMID: 2759 amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB); 2760 break; 2761 default: 2762 return -EINVAL; 2763 } 2764 2765 return 0; 2766 } 2767