1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <drm/drmP.h> 29 #include <drm/amdgpu_drm.h> 30 #include "amdgpu.h" 31 #include "amdgpu_trace.h" 32 33 /* 34 * GPUVM 35 * GPUVM is similar to the legacy gart on older asics, however 36 * rather than there being a single global gart table 37 * for the entire GPU, there are multiple VM page tables active 38 * at any given time. The VM page tables can contain a mix 39 * vram pages and system memory pages and system memory pages 40 * can be mapped as snooped (cached system pages) or unsnooped 41 * (uncached system pages). 42 * Each VM has an ID associated with it and there is a page table 43 * associated with each VMID. When execting a command buffer, 44 * the kernel tells the the ring what VMID to use for that command 45 * buffer. VMIDs are allocated dynamically as commands are submitted. 46 * The userspace drivers maintain their own address space and the kernel 47 * sets up their pages tables accordingly when they submit their 48 * command buffers and a VMID is assigned. 49 * Cayman/Trinity support up to 8 active VMs at any given time; 50 * SI supports 16. 51 */ 52 53 /** 54 * amdgpu_vm_num_pde - return the number of page directory entries 55 * 56 * @adev: amdgpu_device pointer 57 * 58 * Calculate the number of page directory entries (cayman+). 59 */ 60 static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev) 61 { 62 return adev->vm_manager.max_pfn >> amdgpu_vm_block_size; 63 } 64 65 /** 66 * amdgpu_vm_directory_size - returns the size of the page directory in bytes 67 * 68 * @adev: amdgpu_device pointer 69 * 70 * Calculate the size of the page directory in bytes (cayman+). 71 */ 72 static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev) 73 { 74 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8); 75 } 76 77 /** 78 * amdgpu_vm_get_bos - add the vm BOs to a validation list 79 * 80 * @vm: vm providing the BOs 81 * @head: head of validation list 82 * 83 * Add the page directory to the list of BOs to 84 * validate for command submission (cayman+). 85 */ 86 struct amdgpu_bo_list_entry *amdgpu_vm_get_bos(struct amdgpu_device *adev, 87 struct amdgpu_vm *vm, 88 struct list_head *head) 89 { 90 struct amdgpu_bo_list_entry *list; 91 unsigned i, idx; 92 93 mutex_lock(&vm->mutex); 94 list = drm_malloc_ab(vm->max_pde_used + 2, 95 sizeof(struct amdgpu_bo_list_entry)); 96 if (!list) { 97 mutex_unlock(&vm->mutex); 98 return NULL; 99 } 100 101 /* add the vm page table to the list */ 102 list[0].robj = vm->page_directory; 103 list[0].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM; 104 list[0].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM; 105 list[0].priority = 0; 106 list[0].tv.bo = &vm->page_directory->tbo; 107 list[0].tv.shared = true; 108 list_add(&list[0].tv.head, head); 109 110 for (i = 0, idx = 1; i <= vm->max_pde_used; i++) { 111 if (!vm->page_tables[i].bo) 112 continue; 113 114 list[idx].robj = vm->page_tables[i].bo; 115 list[idx].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM; 116 list[idx].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM; 117 list[idx].priority = 0; 118 list[idx].tv.bo = &list[idx].robj->tbo; 119 list[idx].tv.shared = true; 120 list_add(&list[idx++].tv.head, head); 121 } 122 mutex_unlock(&vm->mutex); 123 124 return list; 125 } 126 127 /** 128 * amdgpu_vm_grab_id - allocate the next free VMID 129 * 130 * @vm: vm to allocate id for 131 * @ring: ring we want to submit job to 132 * @sync: sync object where we add dependencies 133 * 134 * Allocate an id for the vm, adding fences to the sync obj as necessary. 135 * 136 * Global mutex must be locked! 137 */ 138 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, 139 struct amdgpu_sync *sync) 140 { 141 struct amdgpu_fence *best[AMDGPU_MAX_RINGS] = {}; 142 struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; 143 struct amdgpu_device *adev = ring->adev; 144 145 unsigned choices[2] = {}; 146 unsigned i; 147 148 /* check if the id is still valid */ 149 if (vm_id->id && vm_id->last_id_use && 150 vm_id->last_id_use == adev->vm_manager.active[vm_id->id]) 151 return 0; 152 153 /* we definately need to flush */ 154 vm_id->pd_gpu_addr = ~0ll; 155 156 /* skip over VMID 0, since it is the system VM */ 157 for (i = 1; i < adev->vm_manager.nvm; ++i) { 158 struct amdgpu_fence *fence = adev->vm_manager.active[i]; 159 160 if (fence == NULL) { 161 /* found a free one */ 162 vm_id->id = i; 163 trace_amdgpu_vm_grab_id(i, ring->idx); 164 return 0; 165 } 166 167 if (amdgpu_fence_is_earlier(fence, best[fence->ring->idx])) { 168 best[fence->ring->idx] = fence; 169 choices[fence->ring == ring ? 0 : 1] = i; 170 } 171 } 172 173 for (i = 0; i < 2; ++i) { 174 if (choices[i]) { 175 struct amdgpu_fence *fence; 176 177 fence = adev->vm_manager.active[choices[i]]; 178 vm_id->id = choices[i]; 179 180 trace_amdgpu_vm_grab_id(choices[i], ring->idx); 181 return amdgpu_sync_fence(ring->adev, sync, &fence->base); 182 } 183 } 184 185 /* should never happen */ 186 BUG(); 187 return -EINVAL; 188 } 189 190 /** 191 * amdgpu_vm_flush - hardware flush the vm 192 * 193 * @ring: ring to use for flush 194 * @vm: vm we want to flush 195 * @updates: last vm update that we waited for 196 * 197 * Flush the vm (cayman+). 198 * 199 * Global and local mutex must be locked! 200 */ 201 void amdgpu_vm_flush(struct amdgpu_ring *ring, 202 struct amdgpu_vm *vm, 203 struct fence *updates) 204 { 205 uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory); 206 struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; 207 struct fence *flushed_updates = vm_id->flushed_updates; 208 bool is_earlier = false; 209 210 if (flushed_updates && updates) { 211 BUG_ON(flushed_updates->context != updates->context); 212 is_earlier = (updates->seqno - flushed_updates->seqno <= 213 INT_MAX) ? true : false; 214 } 215 216 if (pd_addr != vm_id->pd_gpu_addr || !flushed_updates || 217 is_earlier) { 218 219 trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id->id); 220 if (is_earlier) { 221 vm_id->flushed_updates = fence_get(updates); 222 fence_put(flushed_updates); 223 } 224 if (!flushed_updates) 225 vm_id->flushed_updates = fence_get(updates); 226 vm_id->pd_gpu_addr = pd_addr; 227 amdgpu_ring_emit_vm_flush(ring, vm_id->id, vm_id->pd_gpu_addr); 228 } 229 } 230 231 /** 232 * amdgpu_vm_fence - remember fence for vm 233 * 234 * @adev: amdgpu_device pointer 235 * @vm: vm we want to fence 236 * @fence: fence to remember 237 * 238 * Fence the vm (cayman+). 239 * Set the fence used to protect page table and id. 240 * 241 * Global and local mutex must be locked! 242 */ 243 void amdgpu_vm_fence(struct amdgpu_device *adev, 244 struct amdgpu_vm *vm, 245 struct amdgpu_fence *fence) 246 { 247 unsigned ridx = fence->ring->idx; 248 unsigned vm_id = vm->ids[ridx].id; 249 250 amdgpu_fence_unref(&adev->vm_manager.active[vm_id]); 251 adev->vm_manager.active[vm_id] = amdgpu_fence_ref(fence); 252 253 amdgpu_fence_unref(&vm->ids[ridx].last_id_use); 254 vm->ids[ridx].last_id_use = amdgpu_fence_ref(fence); 255 } 256 257 /** 258 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo 259 * 260 * @vm: requested vm 261 * @bo: requested buffer object 262 * 263 * Find @bo inside the requested vm (cayman+). 264 * Search inside the @bos vm list for the requested vm 265 * Returns the found bo_va or NULL if none is found 266 * 267 * Object has to be reserved! 268 */ 269 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, 270 struct amdgpu_bo *bo) 271 { 272 struct amdgpu_bo_va *bo_va; 273 274 list_for_each_entry(bo_va, &bo->va, bo_list) { 275 if (bo_va->vm == vm) { 276 return bo_va; 277 } 278 } 279 return NULL; 280 } 281 282 /** 283 * amdgpu_vm_update_pages - helper to call the right asic function 284 * 285 * @adev: amdgpu_device pointer 286 * @ib: indirect buffer to fill with commands 287 * @pe: addr of the page entry 288 * @addr: dst addr to write into pe 289 * @count: number of page entries to update 290 * @incr: increase next addr by incr bytes 291 * @flags: hw access flags 292 * @gtt_flags: GTT hw access flags 293 * 294 * Traces the parameters and calls the right asic functions 295 * to setup the page table using the DMA. 296 */ 297 static void amdgpu_vm_update_pages(struct amdgpu_device *adev, 298 struct amdgpu_ib *ib, 299 uint64_t pe, uint64_t addr, 300 unsigned count, uint32_t incr, 301 uint32_t flags, uint32_t gtt_flags) 302 { 303 trace_amdgpu_vm_set_page(pe, addr, count, incr, flags); 304 305 if ((flags & AMDGPU_PTE_SYSTEM) && (flags == gtt_flags)) { 306 uint64_t src = adev->gart.table_addr + (addr >> 12) * 8; 307 amdgpu_vm_copy_pte(adev, ib, pe, src, count); 308 309 } else if ((flags & AMDGPU_PTE_SYSTEM) || (count < 3)) { 310 amdgpu_vm_write_pte(adev, ib, pe, addr, 311 count, incr, flags); 312 313 } else { 314 amdgpu_vm_set_pte_pde(adev, ib, pe, addr, 315 count, incr, flags); 316 } 317 } 318 319 int amdgpu_vm_free_job(struct amdgpu_job *sched_job) 320 { 321 int i; 322 for (i = 0; i < sched_job->num_ibs; i++) 323 amdgpu_ib_free(sched_job->adev, &sched_job->ibs[i]); 324 kfree(sched_job->ibs); 325 return 0; 326 } 327 328 /** 329 * amdgpu_vm_clear_bo - initially clear the page dir/table 330 * 331 * @adev: amdgpu_device pointer 332 * @bo: bo to clear 333 */ 334 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, 335 struct amdgpu_bo *bo) 336 { 337 struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; 338 struct fence *fence = NULL; 339 struct amdgpu_ib *ib; 340 unsigned entries; 341 uint64_t addr; 342 int r; 343 344 r = amdgpu_bo_reserve(bo, false); 345 if (r) 346 return r; 347 348 r = reservation_object_reserve_shared(bo->tbo.resv); 349 if (r) 350 return r; 351 352 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 353 if (r) 354 goto error_unreserve; 355 356 addr = amdgpu_bo_gpu_offset(bo); 357 entries = amdgpu_bo_size(bo) / 8; 358 359 ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); 360 if (!ib) 361 goto error_unreserve; 362 363 r = amdgpu_ib_get(ring, NULL, entries * 2 + 64, ib); 364 if (r) 365 goto error_free; 366 367 ib->length_dw = 0; 368 369 amdgpu_vm_update_pages(adev, ib, addr, 0, entries, 0, 0, 0); 370 amdgpu_vm_pad_ib(adev, ib); 371 WARN_ON(ib->length_dw > 64); 372 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, 373 &amdgpu_vm_free_job, 374 AMDGPU_FENCE_OWNER_VM, 375 &fence); 376 if (!r) 377 amdgpu_bo_fence(bo, fence, true); 378 fence_put(fence); 379 if (amdgpu_enable_scheduler) { 380 amdgpu_bo_unreserve(bo); 381 return 0; 382 } 383 error_free: 384 amdgpu_ib_free(adev, ib); 385 kfree(ib); 386 387 error_unreserve: 388 amdgpu_bo_unreserve(bo); 389 return r; 390 } 391 392 /** 393 * amdgpu_vm_map_gart - get the physical address of a gart page 394 * 395 * @adev: amdgpu_device pointer 396 * @addr: the unmapped addr 397 * 398 * Look up the physical address of the page that the pte resolves 399 * to (cayman+). 400 * Returns the physical address of the page. 401 */ 402 uint64_t amdgpu_vm_map_gart(struct amdgpu_device *adev, uint64_t addr) 403 { 404 uint64_t result; 405 406 /* page table offset */ 407 result = adev->gart.pages_addr[addr >> PAGE_SHIFT]; 408 409 /* in case cpu page size != gpu page size*/ 410 result |= addr & (~PAGE_MASK); 411 412 return result; 413 } 414 415 /** 416 * amdgpu_vm_update_pdes - make sure that page directory is valid 417 * 418 * @adev: amdgpu_device pointer 419 * @vm: requested vm 420 * @start: start of GPU address range 421 * @end: end of GPU address range 422 * 423 * Allocates new page tables if necessary 424 * and updates the page directory (cayman+). 425 * Returns 0 for success, error for failure. 426 * 427 * Global and local mutex must be locked! 428 */ 429 int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, 430 struct amdgpu_vm *vm) 431 { 432 struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; 433 struct amdgpu_bo *pd = vm->page_directory; 434 uint64_t pd_addr = amdgpu_bo_gpu_offset(pd); 435 uint32_t incr = AMDGPU_VM_PTE_COUNT * 8; 436 uint64_t last_pde = ~0, last_pt = ~0; 437 unsigned count = 0, pt_idx, ndw; 438 struct amdgpu_ib *ib; 439 struct fence *fence = NULL; 440 441 int r; 442 443 /* padding, etc. */ 444 ndw = 64; 445 446 /* assume the worst case */ 447 ndw += vm->max_pde_used * 6; 448 449 /* update too big for an IB */ 450 if (ndw > 0xfffff) 451 return -ENOMEM; 452 453 ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); 454 if (!ib) 455 return -ENOMEM; 456 457 r = amdgpu_ib_get(ring, NULL, ndw * 4, ib); 458 if (r) 459 return r; 460 ib->length_dw = 0; 461 462 /* walk over the address space and update the page directory */ 463 for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) { 464 struct amdgpu_bo *bo = vm->page_tables[pt_idx].bo; 465 uint64_t pde, pt; 466 467 if (bo == NULL) 468 continue; 469 470 pt = amdgpu_bo_gpu_offset(bo); 471 if (vm->page_tables[pt_idx].addr == pt) 472 continue; 473 vm->page_tables[pt_idx].addr = pt; 474 475 pde = pd_addr + pt_idx * 8; 476 if (((last_pde + 8 * count) != pde) || 477 ((last_pt + incr * count) != pt)) { 478 479 if (count) { 480 amdgpu_vm_update_pages(adev, ib, last_pde, 481 last_pt, count, incr, 482 AMDGPU_PTE_VALID, 0); 483 } 484 485 count = 1; 486 last_pde = pde; 487 last_pt = pt; 488 } else { 489 ++count; 490 } 491 } 492 493 if (count) 494 amdgpu_vm_update_pages(adev, ib, last_pde, last_pt, count, 495 incr, AMDGPU_PTE_VALID, 0); 496 497 if (ib->length_dw != 0) { 498 amdgpu_vm_pad_ib(adev, ib); 499 amdgpu_sync_resv(adev, &ib->sync, pd->tbo.resv, AMDGPU_FENCE_OWNER_VM); 500 WARN_ON(ib->length_dw > ndw); 501 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, 502 &amdgpu_vm_free_job, 503 AMDGPU_FENCE_OWNER_VM, 504 &fence); 505 if (r) 506 goto error_free; 507 508 amdgpu_bo_fence(pd, fence, true); 509 fence_put(vm->page_directory_fence); 510 vm->page_directory_fence = fence_get(fence); 511 fence_put(fence); 512 } 513 514 if (!amdgpu_enable_scheduler || ib->length_dw == 0) { 515 amdgpu_ib_free(adev, ib); 516 kfree(ib); 517 } 518 519 return 0; 520 521 error_free: 522 amdgpu_ib_free(adev, ib); 523 kfree(ib); 524 return r; 525 } 526 527 /** 528 * amdgpu_vm_frag_ptes - add fragment information to PTEs 529 * 530 * @adev: amdgpu_device pointer 531 * @ib: IB for the update 532 * @pe_start: first PTE to handle 533 * @pe_end: last PTE to handle 534 * @addr: addr those PTEs should point to 535 * @flags: hw mapping flags 536 * @gtt_flags: GTT hw mapping flags 537 * 538 * Global and local mutex must be locked! 539 */ 540 static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev, 541 struct amdgpu_ib *ib, 542 uint64_t pe_start, uint64_t pe_end, 543 uint64_t addr, uint32_t flags, 544 uint32_t gtt_flags) 545 { 546 /** 547 * The MC L1 TLB supports variable sized pages, based on a fragment 548 * field in the PTE. When this field is set to a non-zero value, page 549 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE 550 * flags are considered valid for all PTEs within the fragment range 551 * and corresponding mappings are assumed to be physically contiguous. 552 * 553 * The L1 TLB can store a single PTE for the whole fragment, 554 * significantly increasing the space available for translation 555 * caching. This leads to large improvements in throughput when the 556 * TLB is under pressure. 557 * 558 * The L2 TLB distributes small and large fragments into two 559 * asymmetric partitions. The large fragment cache is significantly 560 * larger. Thus, we try to use large fragments wherever possible. 561 * Userspace can support this by aligning virtual base address and 562 * allocation size to the fragment size. 563 */ 564 565 /* SI and newer are optimized for 64KB */ 566 uint64_t frag_flags = AMDGPU_PTE_FRAG_64KB; 567 uint64_t frag_align = 0x80; 568 569 uint64_t frag_start = ALIGN(pe_start, frag_align); 570 uint64_t frag_end = pe_end & ~(frag_align - 1); 571 572 unsigned count; 573 574 /* system pages are non continuously */ 575 if ((flags & AMDGPU_PTE_SYSTEM) || !(flags & AMDGPU_PTE_VALID) || 576 (frag_start >= frag_end)) { 577 578 count = (pe_end - pe_start) / 8; 579 amdgpu_vm_update_pages(adev, ib, pe_start, addr, count, 580 AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); 581 return; 582 } 583 584 /* handle the 4K area at the beginning */ 585 if (pe_start != frag_start) { 586 count = (frag_start - pe_start) / 8; 587 amdgpu_vm_update_pages(adev, ib, pe_start, addr, count, 588 AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); 589 addr += AMDGPU_GPU_PAGE_SIZE * count; 590 } 591 592 /* handle the area in the middle */ 593 count = (frag_end - frag_start) / 8; 594 amdgpu_vm_update_pages(adev, ib, frag_start, addr, count, 595 AMDGPU_GPU_PAGE_SIZE, flags | frag_flags, 596 gtt_flags); 597 598 /* handle the 4K area at the end */ 599 if (frag_end != pe_end) { 600 addr += AMDGPU_GPU_PAGE_SIZE * count; 601 count = (pe_end - frag_end) / 8; 602 amdgpu_vm_update_pages(adev, ib, frag_end, addr, count, 603 AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); 604 } 605 } 606 607 /** 608 * amdgpu_vm_update_ptes - make sure that page tables are valid 609 * 610 * @adev: amdgpu_device pointer 611 * @vm: requested vm 612 * @start: start of GPU address range 613 * @end: end of GPU address range 614 * @dst: destination address to map to 615 * @flags: mapping flags 616 * 617 * Update the page tables in the range @start - @end (cayman+). 618 * 619 * Global and local mutex must be locked! 620 */ 621 static int amdgpu_vm_update_ptes(struct amdgpu_device *adev, 622 struct amdgpu_vm *vm, 623 struct amdgpu_ib *ib, 624 uint64_t start, uint64_t end, 625 uint64_t dst, uint32_t flags, 626 uint32_t gtt_flags) 627 { 628 uint64_t mask = AMDGPU_VM_PTE_COUNT - 1; 629 uint64_t last_pte = ~0, last_dst = ~0; 630 void *owner = AMDGPU_FENCE_OWNER_VM; 631 unsigned count = 0; 632 uint64_t addr; 633 634 /* sync to everything on unmapping */ 635 if (!(flags & AMDGPU_PTE_VALID)) 636 owner = AMDGPU_FENCE_OWNER_UNDEFINED; 637 638 /* walk over the address space and update the page tables */ 639 for (addr = start; addr < end; ) { 640 uint64_t pt_idx = addr >> amdgpu_vm_block_size; 641 struct amdgpu_bo *pt = vm->page_tables[pt_idx].bo; 642 unsigned nptes; 643 uint64_t pte; 644 int r; 645 646 amdgpu_sync_resv(adev, &ib->sync, pt->tbo.resv, owner); 647 r = reservation_object_reserve_shared(pt->tbo.resv); 648 if (r) 649 return r; 650 651 if ((addr & ~mask) == (end & ~mask)) 652 nptes = end - addr; 653 else 654 nptes = AMDGPU_VM_PTE_COUNT - (addr & mask); 655 656 pte = amdgpu_bo_gpu_offset(pt); 657 pte += (addr & mask) * 8; 658 659 if ((last_pte + 8 * count) != pte) { 660 661 if (count) { 662 amdgpu_vm_frag_ptes(adev, ib, last_pte, 663 last_pte + 8 * count, 664 last_dst, flags, 665 gtt_flags); 666 } 667 668 count = nptes; 669 last_pte = pte; 670 last_dst = dst; 671 } else { 672 count += nptes; 673 } 674 675 addr += nptes; 676 dst += nptes * AMDGPU_GPU_PAGE_SIZE; 677 } 678 679 if (count) { 680 amdgpu_vm_frag_ptes(adev, ib, last_pte, 681 last_pte + 8 * count, 682 last_dst, flags, gtt_flags); 683 } 684 685 return 0; 686 } 687 688 /** 689 * amdgpu_vm_fence_pts - fence page tables after an update 690 * 691 * @vm: requested vm 692 * @start: start of GPU address range 693 * @end: end of GPU address range 694 * @fence: fence to use 695 * 696 * Fence the page tables in the range @start - @end (cayman+). 697 * 698 * Global and local mutex must be locked! 699 */ 700 static void amdgpu_vm_fence_pts(struct amdgpu_vm *vm, 701 uint64_t start, uint64_t end, 702 struct fence *fence) 703 { 704 unsigned i; 705 706 start >>= amdgpu_vm_block_size; 707 end >>= amdgpu_vm_block_size; 708 709 for (i = start; i <= end; ++i) 710 amdgpu_bo_fence(vm->page_tables[i].bo, fence, true); 711 } 712 713 /** 714 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table 715 * 716 * @adev: amdgpu_device pointer 717 * @vm: requested vm 718 * @mapping: mapped range and flags to use for the update 719 * @addr: addr to set the area to 720 * @gtt_flags: flags as they are used for GTT 721 * @fence: optional resulting fence 722 * 723 * Fill in the page table entries for @mapping. 724 * Returns 0 for success, -EINVAL for failure. 725 * 726 * Object have to be reserved and mutex must be locked! 727 */ 728 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, 729 struct amdgpu_vm *vm, 730 struct amdgpu_bo_va_mapping *mapping, 731 uint64_t addr, uint32_t gtt_flags, 732 struct fence **fence) 733 { 734 struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; 735 unsigned nptes, ncmds, ndw; 736 uint32_t flags = gtt_flags; 737 struct amdgpu_ib *ib; 738 struct fence *f = NULL; 739 int r; 740 741 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here 742 * but in case of something, we filter the flags in first place 743 */ 744 if (!(mapping->flags & AMDGPU_PTE_READABLE)) 745 flags &= ~AMDGPU_PTE_READABLE; 746 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) 747 flags &= ~AMDGPU_PTE_WRITEABLE; 748 749 trace_amdgpu_vm_bo_update(mapping); 750 751 nptes = mapping->it.last - mapping->it.start + 1; 752 753 /* 754 * reserve space for one command every (1 << BLOCK_SIZE) 755 * entries or 2k dwords (whatever is smaller) 756 */ 757 ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1; 758 759 /* padding, etc. */ 760 ndw = 64; 761 762 if ((flags & AMDGPU_PTE_SYSTEM) && (flags == gtt_flags)) { 763 /* only copy commands needed */ 764 ndw += ncmds * 7; 765 766 } else if (flags & AMDGPU_PTE_SYSTEM) { 767 /* header for write data commands */ 768 ndw += ncmds * 4; 769 770 /* body of write data command */ 771 ndw += nptes * 2; 772 773 } else { 774 /* set page commands needed */ 775 ndw += ncmds * 10; 776 777 /* two extra commands for begin/end of fragment */ 778 ndw += 2 * 10; 779 } 780 781 /* update too big for an IB */ 782 if (ndw > 0xfffff) 783 return -ENOMEM; 784 785 ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); 786 if (!ib) 787 return -ENOMEM; 788 789 r = amdgpu_ib_get(ring, NULL, ndw * 4, ib); 790 if (r) { 791 kfree(ib); 792 return r; 793 } 794 795 ib->length_dw = 0; 796 797 r = amdgpu_vm_update_ptes(adev, vm, ib, mapping->it.start, 798 mapping->it.last + 1, addr + mapping->offset, 799 flags, gtt_flags); 800 801 if (r) { 802 amdgpu_ib_free(adev, ib); 803 kfree(ib); 804 return r; 805 } 806 807 amdgpu_vm_pad_ib(adev, ib); 808 WARN_ON(ib->length_dw > ndw); 809 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, 810 &amdgpu_vm_free_job, 811 AMDGPU_FENCE_OWNER_VM, 812 &f); 813 if (r) 814 goto error_free; 815 816 amdgpu_vm_fence_pts(vm, mapping->it.start, 817 mapping->it.last + 1, f); 818 if (fence) { 819 fence_put(*fence); 820 *fence = fence_get(f); 821 } 822 fence_put(f); 823 if (!amdgpu_enable_scheduler) { 824 amdgpu_ib_free(adev, ib); 825 kfree(ib); 826 } 827 return 0; 828 829 error_free: 830 amdgpu_ib_free(adev, ib); 831 kfree(ib); 832 return r; 833 } 834 835 /** 836 * amdgpu_vm_bo_update - update all BO mappings in the vm page table 837 * 838 * @adev: amdgpu_device pointer 839 * @bo_va: requested BO and VM object 840 * @mem: ttm mem 841 * 842 * Fill in the page table entries for @bo_va. 843 * Returns 0 for success, -EINVAL for failure. 844 * 845 * Object have to be reserved and mutex must be locked! 846 */ 847 int amdgpu_vm_bo_update(struct amdgpu_device *adev, 848 struct amdgpu_bo_va *bo_va, 849 struct ttm_mem_reg *mem) 850 { 851 struct amdgpu_vm *vm = bo_va->vm; 852 struct amdgpu_bo_va_mapping *mapping; 853 uint32_t flags; 854 uint64_t addr; 855 int r; 856 857 if (mem) { 858 addr = mem->start << PAGE_SHIFT; 859 if (mem->mem_type != TTM_PL_TT) 860 addr += adev->vm_manager.vram_base_offset; 861 } else { 862 addr = 0; 863 } 864 865 flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem); 866 867 spin_lock(&vm->status_lock); 868 if (!list_empty(&bo_va->vm_status)) 869 list_splice_init(&bo_va->valids, &bo_va->invalids); 870 spin_unlock(&vm->status_lock); 871 872 list_for_each_entry(mapping, &bo_va->invalids, list) { 873 r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, addr, 874 flags, &bo_va->last_pt_update); 875 if (r) 876 return r; 877 } 878 879 spin_lock(&vm->status_lock); 880 list_splice_init(&bo_va->invalids, &bo_va->valids); 881 list_del_init(&bo_va->vm_status); 882 if (!mem) 883 list_add(&bo_va->vm_status, &vm->cleared); 884 spin_unlock(&vm->status_lock); 885 886 return 0; 887 } 888 889 /** 890 * amdgpu_vm_clear_freed - clear freed BOs in the PT 891 * 892 * @adev: amdgpu_device pointer 893 * @vm: requested vm 894 * 895 * Make sure all freed BOs are cleared in the PT. 896 * Returns 0 for success. 897 * 898 * PTs have to be reserved and mutex must be locked! 899 */ 900 int amdgpu_vm_clear_freed(struct amdgpu_device *adev, 901 struct amdgpu_vm *vm) 902 { 903 struct amdgpu_bo_va_mapping *mapping; 904 int r; 905 906 while (!list_empty(&vm->freed)) { 907 mapping = list_first_entry(&vm->freed, 908 struct amdgpu_bo_va_mapping, list); 909 list_del(&mapping->list); 910 911 r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, 0, 0, NULL); 912 kfree(mapping); 913 if (r) 914 return r; 915 916 } 917 return 0; 918 919 } 920 921 /** 922 * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT 923 * 924 * @adev: amdgpu_device pointer 925 * @vm: requested vm 926 * 927 * Make sure all invalidated BOs are cleared in the PT. 928 * Returns 0 for success. 929 * 930 * PTs have to be reserved and mutex must be locked! 931 */ 932 int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, 933 struct amdgpu_vm *vm, struct amdgpu_sync *sync) 934 { 935 struct amdgpu_bo_va *bo_va = NULL; 936 int r = 0; 937 938 spin_lock(&vm->status_lock); 939 while (!list_empty(&vm->invalidated)) { 940 bo_va = list_first_entry(&vm->invalidated, 941 struct amdgpu_bo_va, vm_status); 942 spin_unlock(&vm->status_lock); 943 944 r = amdgpu_vm_bo_update(adev, bo_va, NULL); 945 if (r) 946 return r; 947 948 spin_lock(&vm->status_lock); 949 } 950 spin_unlock(&vm->status_lock); 951 952 if (bo_va) 953 r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update); 954 955 return r; 956 } 957 958 /** 959 * amdgpu_vm_bo_add - add a bo to a specific vm 960 * 961 * @adev: amdgpu_device pointer 962 * @vm: requested vm 963 * @bo: amdgpu buffer object 964 * 965 * Add @bo into the requested vm (cayman+). 966 * Add @bo to the list of bos associated with the vm 967 * Returns newly added bo_va or NULL for failure 968 * 969 * Object has to be reserved! 970 */ 971 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, 972 struct amdgpu_vm *vm, 973 struct amdgpu_bo *bo) 974 { 975 struct amdgpu_bo_va *bo_va; 976 977 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL); 978 if (bo_va == NULL) { 979 return NULL; 980 } 981 bo_va->vm = vm; 982 bo_va->bo = bo; 983 bo_va->ref_count = 1; 984 INIT_LIST_HEAD(&bo_va->bo_list); 985 INIT_LIST_HEAD(&bo_va->valids); 986 INIT_LIST_HEAD(&bo_va->invalids); 987 INIT_LIST_HEAD(&bo_va->vm_status); 988 989 mutex_lock(&vm->mutex); 990 list_add_tail(&bo_va->bo_list, &bo->va); 991 mutex_unlock(&vm->mutex); 992 993 return bo_va; 994 } 995 996 /** 997 * amdgpu_vm_bo_map - map bo inside a vm 998 * 999 * @adev: amdgpu_device pointer 1000 * @bo_va: bo_va to store the address 1001 * @saddr: where to map the BO 1002 * @offset: requested offset in the BO 1003 * @flags: attributes of pages (read/write/valid/etc.) 1004 * 1005 * Add a mapping of the BO at the specefied addr into the VM. 1006 * Returns 0 for success, error for failure. 1007 * 1008 * Object has to be reserved and gets unreserved by this function! 1009 */ 1010 int amdgpu_vm_bo_map(struct amdgpu_device *adev, 1011 struct amdgpu_bo_va *bo_va, 1012 uint64_t saddr, uint64_t offset, 1013 uint64_t size, uint32_t flags) 1014 { 1015 struct amdgpu_bo_va_mapping *mapping; 1016 struct amdgpu_vm *vm = bo_va->vm; 1017 struct interval_tree_node *it; 1018 unsigned last_pfn, pt_idx; 1019 uint64_t eaddr; 1020 int r; 1021 1022 /* validate the parameters */ 1023 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || 1024 size == 0 || size & AMDGPU_GPU_PAGE_MASK) { 1025 amdgpu_bo_unreserve(bo_va->bo); 1026 return -EINVAL; 1027 } 1028 1029 /* make sure object fit at this offset */ 1030 eaddr = saddr + size; 1031 if ((saddr >= eaddr) || (offset + size > amdgpu_bo_size(bo_va->bo))) { 1032 amdgpu_bo_unreserve(bo_va->bo); 1033 return -EINVAL; 1034 } 1035 1036 last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE; 1037 if (last_pfn > adev->vm_manager.max_pfn) { 1038 dev_err(adev->dev, "va above limit (0x%08X > 0x%08X)\n", 1039 last_pfn, adev->vm_manager.max_pfn); 1040 amdgpu_bo_unreserve(bo_va->bo); 1041 return -EINVAL; 1042 } 1043 1044 mutex_lock(&vm->mutex); 1045 1046 saddr /= AMDGPU_GPU_PAGE_SIZE; 1047 eaddr /= AMDGPU_GPU_PAGE_SIZE; 1048 1049 it = interval_tree_iter_first(&vm->va, saddr, eaddr - 1); 1050 if (it) { 1051 struct amdgpu_bo_va_mapping *tmp; 1052 tmp = container_of(it, struct amdgpu_bo_va_mapping, it); 1053 /* bo and tmp overlap, invalid addr */ 1054 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with " 1055 "0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr, 1056 tmp->it.start, tmp->it.last + 1); 1057 amdgpu_bo_unreserve(bo_va->bo); 1058 r = -EINVAL; 1059 goto error_unlock; 1060 } 1061 1062 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 1063 if (!mapping) { 1064 amdgpu_bo_unreserve(bo_va->bo); 1065 r = -ENOMEM; 1066 goto error_unlock; 1067 } 1068 1069 INIT_LIST_HEAD(&mapping->list); 1070 mapping->it.start = saddr; 1071 mapping->it.last = eaddr - 1; 1072 mapping->offset = offset; 1073 mapping->flags = flags; 1074 1075 list_add(&mapping->list, &bo_va->invalids); 1076 interval_tree_insert(&mapping->it, &vm->va); 1077 trace_amdgpu_vm_bo_map(bo_va, mapping); 1078 1079 /* Make sure the page tables are allocated */ 1080 saddr >>= amdgpu_vm_block_size; 1081 eaddr >>= amdgpu_vm_block_size; 1082 1083 BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev)); 1084 1085 if (eaddr > vm->max_pde_used) 1086 vm->max_pde_used = eaddr; 1087 1088 amdgpu_bo_unreserve(bo_va->bo); 1089 1090 /* walk over the address space and allocate the page tables */ 1091 for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) { 1092 struct amdgpu_bo *pt; 1093 1094 if (vm->page_tables[pt_idx].bo) 1095 continue; 1096 1097 /* drop mutex to allocate and clear page table */ 1098 mutex_unlock(&vm->mutex); 1099 1100 r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8, 1101 AMDGPU_GPU_PAGE_SIZE, true, 1102 AMDGPU_GEM_DOMAIN_VRAM, 1103 AMDGPU_GEM_CREATE_NO_CPU_ACCESS, 1104 NULL, &pt); 1105 if (r) 1106 goto error_free; 1107 1108 r = amdgpu_vm_clear_bo(adev, pt); 1109 if (r) { 1110 amdgpu_bo_unref(&pt); 1111 goto error_free; 1112 } 1113 1114 /* aquire mutex again */ 1115 mutex_lock(&vm->mutex); 1116 if (vm->page_tables[pt_idx].bo) { 1117 /* someone else allocated the pt in the meantime */ 1118 mutex_unlock(&vm->mutex); 1119 amdgpu_bo_unref(&pt); 1120 mutex_lock(&vm->mutex); 1121 continue; 1122 } 1123 1124 vm->page_tables[pt_idx].addr = 0; 1125 vm->page_tables[pt_idx].bo = pt; 1126 } 1127 1128 mutex_unlock(&vm->mutex); 1129 return 0; 1130 1131 error_free: 1132 mutex_lock(&vm->mutex); 1133 list_del(&mapping->list); 1134 interval_tree_remove(&mapping->it, &vm->va); 1135 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1136 kfree(mapping); 1137 1138 error_unlock: 1139 mutex_unlock(&vm->mutex); 1140 return r; 1141 } 1142 1143 /** 1144 * amdgpu_vm_bo_unmap - remove bo mapping from vm 1145 * 1146 * @adev: amdgpu_device pointer 1147 * @bo_va: bo_va to remove the address from 1148 * @saddr: where to the BO is mapped 1149 * 1150 * Remove a mapping of the BO at the specefied addr from the VM. 1151 * Returns 0 for success, error for failure. 1152 * 1153 * Object has to be reserved and gets unreserved by this function! 1154 */ 1155 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, 1156 struct amdgpu_bo_va *bo_va, 1157 uint64_t saddr) 1158 { 1159 struct amdgpu_bo_va_mapping *mapping; 1160 struct amdgpu_vm *vm = bo_va->vm; 1161 bool valid = true; 1162 1163 saddr /= AMDGPU_GPU_PAGE_SIZE; 1164 1165 list_for_each_entry(mapping, &bo_va->valids, list) { 1166 if (mapping->it.start == saddr) 1167 break; 1168 } 1169 1170 if (&mapping->list == &bo_va->valids) { 1171 valid = false; 1172 1173 list_for_each_entry(mapping, &bo_va->invalids, list) { 1174 if (mapping->it.start == saddr) 1175 break; 1176 } 1177 1178 if (&mapping->list == &bo_va->invalids) { 1179 amdgpu_bo_unreserve(bo_va->bo); 1180 return -ENOENT; 1181 } 1182 } 1183 1184 mutex_lock(&vm->mutex); 1185 list_del(&mapping->list); 1186 interval_tree_remove(&mapping->it, &vm->va); 1187 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1188 1189 if (valid) 1190 list_add(&mapping->list, &vm->freed); 1191 else 1192 kfree(mapping); 1193 mutex_unlock(&vm->mutex); 1194 amdgpu_bo_unreserve(bo_va->bo); 1195 1196 return 0; 1197 } 1198 1199 /** 1200 * amdgpu_vm_bo_rmv - remove a bo to a specific vm 1201 * 1202 * @adev: amdgpu_device pointer 1203 * @bo_va: requested bo_va 1204 * 1205 * Remove @bo_va->bo from the requested vm (cayman+). 1206 * 1207 * Object have to be reserved! 1208 */ 1209 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, 1210 struct amdgpu_bo_va *bo_va) 1211 { 1212 struct amdgpu_bo_va_mapping *mapping, *next; 1213 struct amdgpu_vm *vm = bo_va->vm; 1214 1215 list_del(&bo_va->bo_list); 1216 1217 mutex_lock(&vm->mutex); 1218 1219 spin_lock(&vm->status_lock); 1220 list_del(&bo_va->vm_status); 1221 spin_unlock(&vm->status_lock); 1222 1223 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 1224 list_del(&mapping->list); 1225 interval_tree_remove(&mapping->it, &vm->va); 1226 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1227 list_add(&mapping->list, &vm->freed); 1228 } 1229 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { 1230 list_del(&mapping->list); 1231 interval_tree_remove(&mapping->it, &vm->va); 1232 kfree(mapping); 1233 } 1234 1235 fence_put(bo_va->last_pt_update); 1236 kfree(bo_va); 1237 1238 mutex_unlock(&vm->mutex); 1239 } 1240 1241 /** 1242 * amdgpu_vm_bo_invalidate - mark the bo as invalid 1243 * 1244 * @adev: amdgpu_device pointer 1245 * @vm: requested vm 1246 * @bo: amdgpu buffer object 1247 * 1248 * Mark @bo as invalid (cayman+). 1249 */ 1250 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, 1251 struct amdgpu_bo *bo) 1252 { 1253 struct amdgpu_bo_va *bo_va; 1254 1255 list_for_each_entry(bo_va, &bo->va, bo_list) { 1256 spin_lock(&bo_va->vm->status_lock); 1257 if (list_empty(&bo_va->vm_status)) 1258 list_add(&bo_va->vm_status, &bo_va->vm->invalidated); 1259 spin_unlock(&bo_va->vm->status_lock); 1260 } 1261 } 1262 1263 /** 1264 * amdgpu_vm_init - initialize a vm instance 1265 * 1266 * @adev: amdgpu_device pointer 1267 * @vm: requested vm 1268 * 1269 * Init @vm fields (cayman+). 1270 */ 1271 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1272 { 1273 const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE, 1274 AMDGPU_VM_PTE_COUNT * 8); 1275 unsigned pd_size, pd_entries, pts_size; 1276 int i, r; 1277 1278 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 1279 vm->ids[i].id = 0; 1280 vm->ids[i].flushed_updates = NULL; 1281 vm->ids[i].last_id_use = NULL; 1282 } 1283 mutex_init(&vm->mutex); 1284 vm->va = RB_ROOT; 1285 spin_lock_init(&vm->status_lock); 1286 INIT_LIST_HEAD(&vm->invalidated); 1287 INIT_LIST_HEAD(&vm->cleared); 1288 INIT_LIST_HEAD(&vm->freed); 1289 1290 pd_size = amdgpu_vm_directory_size(adev); 1291 pd_entries = amdgpu_vm_num_pdes(adev); 1292 1293 /* allocate page table array */ 1294 pts_size = pd_entries * sizeof(struct amdgpu_vm_pt); 1295 vm->page_tables = kzalloc(pts_size, GFP_KERNEL); 1296 if (vm->page_tables == NULL) { 1297 DRM_ERROR("Cannot allocate memory for page table array\n"); 1298 return -ENOMEM; 1299 } 1300 1301 vm->page_directory_fence = NULL; 1302 1303 r = amdgpu_bo_create(adev, pd_size, align, true, 1304 AMDGPU_GEM_DOMAIN_VRAM, 1305 AMDGPU_GEM_CREATE_NO_CPU_ACCESS, 1306 NULL, &vm->page_directory); 1307 if (r) 1308 return r; 1309 1310 r = amdgpu_vm_clear_bo(adev, vm->page_directory); 1311 if (r) { 1312 amdgpu_bo_unref(&vm->page_directory); 1313 vm->page_directory = NULL; 1314 return r; 1315 } 1316 1317 return 0; 1318 } 1319 1320 /** 1321 * amdgpu_vm_fini - tear down a vm instance 1322 * 1323 * @adev: amdgpu_device pointer 1324 * @vm: requested vm 1325 * 1326 * Tear down @vm (cayman+). 1327 * Unbind the VM and remove all bos from the vm bo list 1328 */ 1329 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1330 { 1331 struct amdgpu_bo_va_mapping *mapping, *tmp; 1332 int i; 1333 1334 if (!RB_EMPTY_ROOT(&vm->va)) { 1335 dev_err(adev->dev, "still active bo inside vm\n"); 1336 } 1337 rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) { 1338 list_del(&mapping->list); 1339 interval_tree_remove(&mapping->it, &vm->va); 1340 kfree(mapping); 1341 } 1342 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { 1343 list_del(&mapping->list); 1344 kfree(mapping); 1345 } 1346 1347 for (i = 0; i < amdgpu_vm_num_pdes(adev); i++) 1348 amdgpu_bo_unref(&vm->page_tables[i].bo); 1349 kfree(vm->page_tables); 1350 1351 amdgpu_bo_unref(&vm->page_directory); 1352 fence_put(vm->page_directory_fence); 1353 1354 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 1355 fence_put(vm->ids[i].flushed_updates); 1356 amdgpu_fence_unref(&vm->ids[i].last_id_use); 1357 } 1358 1359 mutex_destroy(&vm->mutex); 1360 } 1361