1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <drm/drmP.h> 29 #include <drm/amdgpu_drm.h> 30 #include "amdgpu.h" 31 #include "amdgpu_trace.h" 32 33 /* 34 * GPUVM 35 * GPUVM is similar to the legacy gart on older asics, however 36 * rather than there being a single global gart table 37 * for the entire GPU, there are multiple VM page tables active 38 * at any given time. The VM page tables can contain a mix 39 * vram pages and system memory pages and system memory pages 40 * can be mapped as snooped (cached system pages) or unsnooped 41 * (uncached system pages). 42 * Each VM has an ID associated with it and there is a page table 43 * associated with each VMID. When execting a command buffer, 44 * the kernel tells the the ring what VMID to use for that command 45 * buffer. VMIDs are allocated dynamically as commands are submitted. 46 * The userspace drivers maintain their own address space and the kernel 47 * sets up their pages tables accordingly when they submit their 48 * command buffers and a VMID is assigned. 49 * Cayman/Trinity support up to 8 active VMs at any given time; 50 * SI supports 16. 51 */ 52 53 /** 54 * amdgpu_vm_num_pde - return the number of page directory entries 55 * 56 * @adev: amdgpu_device pointer 57 * 58 * Calculate the number of page directory entries (cayman+). 59 */ 60 static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev) 61 { 62 return adev->vm_manager.max_pfn >> amdgpu_vm_block_size; 63 } 64 65 /** 66 * amdgpu_vm_directory_size - returns the size of the page directory in bytes 67 * 68 * @adev: amdgpu_device pointer 69 * 70 * Calculate the size of the page directory in bytes (cayman+). 71 */ 72 static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev) 73 { 74 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8); 75 } 76 77 /** 78 * amdgpu_vm_get_bos - add the vm BOs to a validation list 79 * 80 * @vm: vm providing the BOs 81 * @head: head of validation list 82 * 83 * Add the page directory to the list of BOs to 84 * validate for command submission (cayman+). 85 */ 86 struct amdgpu_bo_list_entry *amdgpu_vm_get_bos(struct amdgpu_device *adev, 87 struct amdgpu_vm *vm, 88 struct list_head *head) 89 { 90 struct amdgpu_bo_list_entry *list; 91 unsigned i, idx; 92 93 mutex_lock(&vm->mutex); 94 list = drm_malloc_ab(vm->max_pde_used + 2, 95 sizeof(struct amdgpu_bo_list_entry)); 96 if (!list) { 97 mutex_unlock(&vm->mutex); 98 return NULL; 99 } 100 101 /* add the vm page table to the list */ 102 list[0].robj = vm->page_directory; 103 list[0].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM; 104 list[0].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM; 105 list[0].priority = 0; 106 list[0].tv.bo = &vm->page_directory->tbo; 107 list[0].tv.shared = true; 108 list_add(&list[0].tv.head, head); 109 110 for (i = 0, idx = 1; i <= vm->max_pde_used; i++) { 111 if (!vm->page_tables[i].bo) 112 continue; 113 114 list[idx].robj = vm->page_tables[i].bo; 115 list[idx].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM; 116 list[idx].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM; 117 list[idx].priority = 0; 118 list[idx].tv.bo = &list[idx].robj->tbo; 119 list[idx].tv.shared = true; 120 list_add(&list[idx++].tv.head, head); 121 } 122 mutex_unlock(&vm->mutex); 123 124 return list; 125 } 126 127 /** 128 * amdgpu_vm_grab_id - allocate the next free VMID 129 * 130 * @vm: vm to allocate id for 131 * @ring: ring we want to submit job to 132 * @sync: sync object where we add dependencies 133 * 134 * Allocate an id for the vm, adding fences to the sync obj as necessary. 135 * 136 * Global mutex must be locked! 137 */ 138 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, 139 struct amdgpu_sync *sync) 140 { 141 struct amdgpu_fence *best[AMDGPU_MAX_RINGS] = {}; 142 struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; 143 struct amdgpu_device *adev = ring->adev; 144 145 unsigned choices[2] = {}; 146 unsigned i; 147 148 /* check if the id is still valid */ 149 if (vm_id->id && vm_id->last_id_use && 150 vm_id->last_id_use == adev->vm_manager.active[vm_id->id]) 151 return 0; 152 153 /* we definately need to flush */ 154 vm_id->pd_gpu_addr = ~0ll; 155 156 /* skip over VMID 0, since it is the system VM */ 157 for (i = 1; i < adev->vm_manager.nvm; ++i) { 158 struct amdgpu_fence *fence = adev->vm_manager.active[i]; 159 160 if (fence == NULL) { 161 /* found a free one */ 162 vm_id->id = i; 163 trace_amdgpu_vm_grab_id(i, ring->idx); 164 return 0; 165 } 166 167 if (amdgpu_fence_is_earlier(fence, best[fence->ring->idx])) { 168 best[fence->ring->idx] = fence; 169 choices[fence->ring == ring ? 0 : 1] = i; 170 } 171 } 172 173 for (i = 0; i < 2; ++i) { 174 if (choices[i]) { 175 struct amdgpu_fence *fence; 176 177 fence = adev->vm_manager.active[choices[i]]; 178 vm_id->id = choices[i]; 179 180 trace_amdgpu_vm_grab_id(choices[i], ring->idx); 181 return amdgpu_sync_fence(ring->adev, sync, &fence->base); 182 } 183 } 184 185 /* should never happen */ 186 BUG(); 187 return -EINVAL; 188 } 189 190 /** 191 * amdgpu_vm_flush - hardware flush the vm 192 * 193 * @ring: ring to use for flush 194 * @vm: vm we want to flush 195 * @updates: last vm update that we waited for 196 * 197 * Flush the vm (cayman+). 198 * 199 * Global and local mutex must be locked! 200 */ 201 void amdgpu_vm_flush(struct amdgpu_ring *ring, 202 struct amdgpu_vm *vm, 203 struct fence *updates) 204 { 205 uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory); 206 struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; 207 struct fence *flushed_updates = vm_id->flushed_updates; 208 bool is_earlier = false; 209 210 if (flushed_updates && updates) { 211 BUG_ON(flushed_updates->context != updates->context); 212 is_earlier = (updates->seqno - flushed_updates->seqno <= 213 INT_MAX) ? true : false; 214 } 215 216 if (pd_addr != vm_id->pd_gpu_addr || !flushed_updates || 217 is_earlier) { 218 219 trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id->id); 220 if (is_earlier) { 221 vm_id->flushed_updates = fence_get(updates); 222 fence_put(flushed_updates); 223 } 224 if (!flushed_updates) 225 vm_id->flushed_updates = fence_get(updates); 226 vm_id->pd_gpu_addr = pd_addr; 227 amdgpu_ring_emit_vm_flush(ring, vm_id->id, vm_id->pd_gpu_addr); 228 } 229 } 230 231 /** 232 * amdgpu_vm_fence - remember fence for vm 233 * 234 * @adev: amdgpu_device pointer 235 * @vm: vm we want to fence 236 * @fence: fence to remember 237 * 238 * Fence the vm (cayman+). 239 * Set the fence used to protect page table and id. 240 * 241 * Global and local mutex must be locked! 242 */ 243 void amdgpu_vm_fence(struct amdgpu_device *adev, 244 struct amdgpu_vm *vm, 245 struct amdgpu_fence *fence) 246 { 247 unsigned ridx = fence->ring->idx; 248 unsigned vm_id = vm->ids[ridx].id; 249 250 amdgpu_fence_unref(&adev->vm_manager.active[vm_id]); 251 adev->vm_manager.active[vm_id] = amdgpu_fence_ref(fence); 252 253 amdgpu_fence_unref(&vm->ids[ridx].last_id_use); 254 vm->ids[ridx].last_id_use = amdgpu_fence_ref(fence); 255 } 256 257 /** 258 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo 259 * 260 * @vm: requested vm 261 * @bo: requested buffer object 262 * 263 * Find @bo inside the requested vm (cayman+). 264 * Search inside the @bos vm list for the requested vm 265 * Returns the found bo_va or NULL if none is found 266 * 267 * Object has to be reserved! 268 */ 269 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, 270 struct amdgpu_bo *bo) 271 { 272 struct amdgpu_bo_va *bo_va; 273 274 list_for_each_entry(bo_va, &bo->va, bo_list) { 275 if (bo_va->vm == vm) { 276 return bo_va; 277 } 278 } 279 return NULL; 280 } 281 282 /** 283 * amdgpu_vm_update_pages - helper to call the right asic function 284 * 285 * @adev: amdgpu_device pointer 286 * @ib: indirect buffer to fill with commands 287 * @pe: addr of the page entry 288 * @addr: dst addr to write into pe 289 * @count: number of page entries to update 290 * @incr: increase next addr by incr bytes 291 * @flags: hw access flags 292 * @gtt_flags: GTT hw access flags 293 * 294 * Traces the parameters and calls the right asic functions 295 * to setup the page table using the DMA. 296 */ 297 static void amdgpu_vm_update_pages(struct amdgpu_device *adev, 298 struct amdgpu_ib *ib, 299 uint64_t pe, uint64_t addr, 300 unsigned count, uint32_t incr, 301 uint32_t flags, uint32_t gtt_flags) 302 { 303 trace_amdgpu_vm_set_page(pe, addr, count, incr, flags); 304 305 if ((flags & AMDGPU_PTE_SYSTEM) && (flags == gtt_flags)) { 306 uint64_t src = adev->gart.table_addr + (addr >> 12) * 8; 307 amdgpu_vm_copy_pte(adev, ib, pe, src, count); 308 309 } else if ((flags & AMDGPU_PTE_SYSTEM) || (count < 3)) { 310 amdgpu_vm_write_pte(adev, ib, pe, addr, 311 count, incr, flags); 312 313 } else { 314 amdgpu_vm_set_pte_pde(adev, ib, pe, addr, 315 count, incr, flags); 316 } 317 } 318 319 int amdgpu_vm_free_job(struct amdgpu_job *sched_job) 320 { 321 int i; 322 for (i = 0; i < sched_job->num_ibs; i++) 323 amdgpu_ib_free(sched_job->adev, &sched_job->ibs[i]); 324 kfree(sched_job->ibs); 325 return 0; 326 } 327 328 /** 329 * amdgpu_vm_clear_bo - initially clear the page dir/table 330 * 331 * @adev: amdgpu_device pointer 332 * @bo: bo to clear 333 */ 334 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, 335 struct amdgpu_bo *bo) 336 { 337 struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; 338 struct fence *fence = NULL; 339 struct amdgpu_ib *ib; 340 unsigned entries; 341 uint64_t addr; 342 int r; 343 344 r = amdgpu_bo_reserve(bo, false); 345 if (r) 346 return r; 347 348 r = reservation_object_reserve_shared(bo->tbo.resv); 349 if (r) 350 return r; 351 352 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 353 if (r) 354 goto error_unreserve; 355 356 addr = amdgpu_bo_gpu_offset(bo); 357 entries = amdgpu_bo_size(bo) / 8; 358 359 ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); 360 if (!ib) 361 goto error_unreserve; 362 363 r = amdgpu_ib_get(ring, NULL, entries * 2 + 64, ib); 364 if (r) 365 goto error_free; 366 367 ib->length_dw = 0; 368 369 amdgpu_vm_update_pages(adev, ib, addr, 0, entries, 0, 0, 0); 370 amdgpu_vm_pad_ib(adev, ib); 371 WARN_ON(ib->length_dw > 64); 372 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, 373 &amdgpu_vm_free_job, 374 AMDGPU_FENCE_OWNER_VM, 375 &fence); 376 if (!r) 377 amdgpu_bo_fence(bo, fence, true); 378 fence_put(fence); 379 if (amdgpu_enable_scheduler) { 380 amdgpu_bo_unreserve(bo); 381 return 0; 382 } 383 error_free: 384 amdgpu_ib_free(adev, ib); 385 kfree(ib); 386 387 error_unreserve: 388 amdgpu_bo_unreserve(bo); 389 return r; 390 } 391 392 /** 393 * amdgpu_vm_map_gart - get the physical address of a gart page 394 * 395 * @adev: amdgpu_device pointer 396 * @addr: the unmapped addr 397 * 398 * Look up the physical address of the page that the pte resolves 399 * to (cayman+). 400 * Returns the physical address of the page. 401 */ 402 uint64_t amdgpu_vm_map_gart(struct amdgpu_device *adev, uint64_t addr) 403 { 404 uint64_t result; 405 406 /* page table offset */ 407 result = adev->gart.pages_addr[addr >> PAGE_SHIFT]; 408 409 /* in case cpu page size != gpu page size*/ 410 result |= addr & (~PAGE_MASK); 411 412 return result; 413 } 414 415 /** 416 * amdgpu_vm_update_pdes - make sure that page directory is valid 417 * 418 * @adev: amdgpu_device pointer 419 * @vm: requested vm 420 * @start: start of GPU address range 421 * @end: end of GPU address range 422 * 423 * Allocates new page tables if necessary 424 * and updates the page directory (cayman+). 425 * Returns 0 for success, error for failure. 426 * 427 * Global and local mutex must be locked! 428 */ 429 int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, 430 struct amdgpu_vm *vm) 431 { 432 struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; 433 struct amdgpu_bo *pd = vm->page_directory; 434 uint64_t pd_addr = amdgpu_bo_gpu_offset(pd); 435 uint32_t incr = AMDGPU_VM_PTE_COUNT * 8; 436 uint64_t last_pde = ~0, last_pt = ~0; 437 unsigned count = 0, pt_idx, ndw; 438 struct amdgpu_ib *ib; 439 struct fence *fence = NULL; 440 441 int r; 442 443 /* padding, etc. */ 444 ndw = 64; 445 446 /* assume the worst case */ 447 ndw += vm->max_pde_used * 6; 448 449 /* update too big for an IB */ 450 if (ndw > 0xfffff) 451 return -ENOMEM; 452 453 ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); 454 if (!ib) 455 return -ENOMEM; 456 457 r = amdgpu_ib_get(ring, NULL, ndw * 4, ib); 458 if (r) 459 return r; 460 ib->length_dw = 0; 461 462 /* walk over the address space and update the page directory */ 463 for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) { 464 struct amdgpu_bo *bo = vm->page_tables[pt_idx].bo; 465 uint64_t pde, pt; 466 467 if (bo == NULL) 468 continue; 469 470 pt = amdgpu_bo_gpu_offset(bo); 471 if (vm->page_tables[pt_idx].addr == pt) 472 continue; 473 vm->page_tables[pt_idx].addr = pt; 474 475 pde = pd_addr + pt_idx * 8; 476 if (((last_pde + 8 * count) != pde) || 477 ((last_pt + incr * count) != pt)) { 478 479 if (count) { 480 amdgpu_vm_update_pages(adev, ib, last_pde, 481 last_pt, count, incr, 482 AMDGPU_PTE_VALID, 0); 483 } 484 485 count = 1; 486 last_pde = pde; 487 last_pt = pt; 488 } else { 489 ++count; 490 } 491 } 492 493 if (count) 494 amdgpu_vm_update_pages(adev, ib, last_pde, last_pt, count, 495 incr, AMDGPU_PTE_VALID, 0); 496 497 if (ib->length_dw != 0) { 498 amdgpu_vm_pad_ib(adev, ib); 499 amdgpu_sync_resv(adev, &ib->sync, pd->tbo.resv, AMDGPU_FENCE_OWNER_VM); 500 WARN_ON(ib->length_dw > ndw); 501 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, 502 &amdgpu_vm_free_job, 503 AMDGPU_FENCE_OWNER_VM, 504 &fence); 505 if (r) 506 goto error_free; 507 508 amdgpu_bo_fence(pd, fence, true); 509 fence_put(vm->page_directory_fence); 510 vm->page_directory_fence = fence_get(fence); 511 fence_put(fence); 512 } 513 514 if (!amdgpu_enable_scheduler || ib->length_dw == 0) { 515 amdgpu_ib_free(adev, ib); 516 kfree(ib); 517 } 518 519 return 0; 520 521 error_free: 522 amdgpu_ib_free(adev, ib); 523 kfree(ib); 524 return r; 525 } 526 527 /** 528 * amdgpu_vm_frag_ptes - add fragment information to PTEs 529 * 530 * @adev: amdgpu_device pointer 531 * @ib: IB for the update 532 * @pe_start: first PTE to handle 533 * @pe_end: last PTE to handle 534 * @addr: addr those PTEs should point to 535 * @flags: hw mapping flags 536 * @gtt_flags: GTT hw mapping flags 537 * 538 * Global and local mutex must be locked! 539 */ 540 static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev, 541 struct amdgpu_ib *ib, 542 uint64_t pe_start, uint64_t pe_end, 543 uint64_t addr, uint32_t flags, 544 uint32_t gtt_flags) 545 { 546 /** 547 * The MC L1 TLB supports variable sized pages, based on a fragment 548 * field in the PTE. When this field is set to a non-zero value, page 549 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE 550 * flags are considered valid for all PTEs within the fragment range 551 * and corresponding mappings are assumed to be physically contiguous. 552 * 553 * The L1 TLB can store a single PTE for the whole fragment, 554 * significantly increasing the space available for translation 555 * caching. This leads to large improvements in throughput when the 556 * TLB is under pressure. 557 * 558 * The L2 TLB distributes small and large fragments into two 559 * asymmetric partitions. The large fragment cache is significantly 560 * larger. Thus, we try to use large fragments wherever possible. 561 * Userspace can support this by aligning virtual base address and 562 * allocation size to the fragment size. 563 */ 564 565 /* SI and newer are optimized for 64KB */ 566 uint64_t frag_flags = AMDGPU_PTE_FRAG_64KB; 567 uint64_t frag_align = 0x80; 568 569 uint64_t frag_start = ALIGN(pe_start, frag_align); 570 uint64_t frag_end = pe_end & ~(frag_align - 1); 571 572 unsigned count; 573 574 /* system pages are non continuously */ 575 if ((flags & AMDGPU_PTE_SYSTEM) || !(flags & AMDGPU_PTE_VALID) || 576 (frag_start >= frag_end)) { 577 578 count = (pe_end - pe_start) / 8; 579 amdgpu_vm_update_pages(adev, ib, pe_start, addr, count, 580 AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); 581 return; 582 } 583 584 /* handle the 4K area at the beginning */ 585 if (pe_start != frag_start) { 586 count = (frag_start - pe_start) / 8; 587 amdgpu_vm_update_pages(adev, ib, pe_start, addr, count, 588 AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); 589 addr += AMDGPU_GPU_PAGE_SIZE * count; 590 } 591 592 /* handle the area in the middle */ 593 count = (frag_end - frag_start) / 8; 594 amdgpu_vm_update_pages(adev, ib, frag_start, addr, count, 595 AMDGPU_GPU_PAGE_SIZE, flags | frag_flags, 596 gtt_flags); 597 598 /* handle the 4K area at the end */ 599 if (frag_end != pe_end) { 600 addr += AMDGPU_GPU_PAGE_SIZE * count; 601 count = (pe_end - frag_end) / 8; 602 amdgpu_vm_update_pages(adev, ib, frag_end, addr, count, 603 AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); 604 } 605 } 606 607 /** 608 * amdgpu_vm_update_ptes - make sure that page tables are valid 609 * 610 * @adev: amdgpu_device pointer 611 * @vm: requested vm 612 * @start: start of GPU address range 613 * @end: end of GPU address range 614 * @dst: destination address to map to 615 * @flags: mapping flags 616 * 617 * Update the page tables in the range @start - @end (cayman+). 618 * 619 * Global and local mutex must be locked! 620 */ 621 static int amdgpu_vm_update_ptes(struct amdgpu_device *adev, 622 struct amdgpu_vm *vm, 623 struct amdgpu_ib *ib, 624 uint64_t start, uint64_t end, 625 uint64_t dst, uint32_t flags, 626 uint32_t gtt_flags) 627 { 628 uint64_t mask = AMDGPU_VM_PTE_COUNT - 1; 629 uint64_t last_pte = ~0, last_dst = ~0; 630 unsigned count = 0; 631 uint64_t addr; 632 633 /* walk over the address space and update the page tables */ 634 for (addr = start; addr < end; ) { 635 uint64_t pt_idx = addr >> amdgpu_vm_block_size; 636 struct amdgpu_bo *pt = vm->page_tables[pt_idx].bo; 637 unsigned nptes; 638 uint64_t pte; 639 int r; 640 641 amdgpu_sync_resv(adev, &ib->sync, pt->tbo.resv, 642 AMDGPU_FENCE_OWNER_VM); 643 r = reservation_object_reserve_shared(pt->tbo.resv); 644 if (r) 645 return r; 646 647 if ((addr & ~mask) == (end & ~mask)) 648 nptes = end - addr; 649 else 650 nptes = AMDGPU_VM_PTE_COUNT - (addr & mask); 651 652 pte = amdgpu_bo_gpu_offset(pt); 653 pte += (addr & mask) * 8; 654 655 if ((last_pte + 8 * count) != pte) { 656 657 if (count) { 658 amdgpu_vm_frag_ptes(adev, ib, last_pte, 659 last_pte + 8 * count, 660 last_dst, flags, 661 gtt_flags); 662 } 663 664 count = nptes; 665 last_pte = pte; 666 last_dst = dst; 667 } else { 668 count += nptes; 669 } 670 671 addr += nptes; 672 dst += nptes * AMDGPU_GPU_PAGE_SIZE; 673 } 674 675 if (count) { 676 amdgpu_vm_frag_ptes(adev, ib, last_pte, 677 last_pte + 8 * count, 678 last_dst, flags, gtt_flags); 679 } 680 681 return 0; 682 } 683 684 /** 685 * amdgpu_vm_fence_pts - fence page tables after an update 686 * 687 * @vm: requested vm 688 * @start: start of GPU address range 689 * @end: end of GPU address range 690 * @fence: fence to use 691 * 692 * Fence the page tables in the range @start - @end (cayman+). 693 * 694 * Global and local mutex must be locked! 695 */ 696 static void amdgpu_vm_fence_pts(struct amdgpu_vm *vm, 697 uint64_t start, uint64_t end, 698 struct fence *fence) 699 { 700 unsigned i; 701 702 start >>= amdgpu_vm_block_size; 703 end >>= amdgpu_vm_block_size; 704 705 for (i = start; i <= end; ++i) 706 amdgpu_bo_fence(vm->page_tables[i].bo, fence, true); 707 } 708 709 /** 710 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table 711 * 712 * @adev: amdgpu_device pointer 713 * @vm: requested vm 714 * @mapping: mapped range and flags to use for the update 715 * @addr: addr to set the area to 716 * @gtt_flags: flags as they are used for GTT 717 * @fence: optional resulting fence 718 * 719 * Fill in the page table entries for @mapping. 720 * Returns 0 for success, -EINVAL for failure. 721 * 722 * Object have to be reserved and mutex must be locked! 723 */ 724 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, 725 struct amdgpu_vm *vm, 726 struct amdgpu_bo_va_mapping *mapping, 727 uint64_t addr, uint32_t gtt_flags, 728 struct fence **fence) 729 { 730 struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; 731 unsigned nptes, ncmds, ndw; 732 uint32_t flags = gtt_flags; 733 struct amdgpu_ib *ib; 734 struct fence *f = NULL; 735 int r; 736 737 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here 738 * but in case of something, we filter the flags in first place 739 */ 740 if (!(mapping->flags & AMDGPU_PTE_READABLE)) 741 flags &= ~AMDGPU_PTE_READABLE; 742 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) 743 flags &= ~AMDGPU_PTE_WRITEABLE; 744 745 trace_amdgpu_vm_bo_update(mapping); 746 747 nptes = mapping->it.last - mapping->it.start + 1; 748 749 /* 750 * reserve space for one command every (1 << BLOCK_SIZE) 751 * entries or 2k dwords (whatever is smaller) 752 */ 753 ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1; 754 755 /* padding, etc. */ 756 ndw = 64; 757 758 if ((flags & AMDGPU_PTE_SYSTEM) && (flags == gtt_flags)) { 759 /* only copy commands needed */ 760 ndw += ncmds * 7; 761 762 } else if (flags & AMDGPU_PTE_SYSTEM) { 763 /* header for write data commands */ 764 ndw += ncmds * 4; 765 766 /* body of write data command */ 767 ndw += nptes * 2; 768 769 } else { 770 /* set page commands needed */ 771 ndw += ncmds * 10; 772 773 /* two extra commands for begin/end of fragment */ 774 ndw += 2 * 10; 775 } 776 777 /* update too big for an IB */ 778 if (ndw > 0xfffff) 779 return -ENOMEM; 780 781 ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); 782 if (!ib) 783 return -ENOMEM; 784 785 r = amdgpu_ib_get(ring, NULL, ndw * 4, ib); 786 if (r) { 787 kfree(ib); 788 return r; 789 } 790 791 ib->length_dw = 0; 792 793 if (!(flags & AMDGPU_PTE_VALID)) { 794 unsigned i; 795 796 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 797 struct amdgpu_fence *f = vm->ids[i].last_id_use; 798 r = amdgpu_sync_fence(adev, &ib->sync, &f->base); 799 if (r) 800 return r; 801 } 802 } 803 804 r = amdgpu_vm_update_ptes(adev, vm, ib, mapping->it.start, 805 mapping->it.last + 1, addr + mapping->offset, 806 flags, gtt_flags); 807 808 if (r) { 809 amdgpu_ib_free(adev, ib); 810 kfree(ib); 811 return r; 812 } 813 814 amdgpu_vm_pad_ib(adev, ib); 815 WARN_ON(ib->length_dw > ndw); 816 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, 817 &amdgpu_vm_free_job, 818 AMDGPU_FENCE_OWNER_VM, 819 &f); 820 if (r) 821 goto error_free; 822 823 amdgpu_vm_fence_pts(vm, mapping->it.start, 824 mapping->it.last + 1, f); 825 if (fence) { 826 fence_put(*fence); 827 *fence = fence_get(f); 828 } 829 fence_put(f); 830 if (!amdgpu_enable_scheduler) { 831 amdgpu_ib_free(adev, ib); 832 kfree(ib); 833 } 834 return 0; 835 836 error_free: 837 amdgpu_ib_free(adev, ib); 838 kfree(ib); 839 return r; 840 } 841 842 /** 843 * amdgpu_vm_bo_update - update all BO mappings in the vm page table 844 * 845 * @adev: amdgpu_device pointer 846 * @bo_va: requested BO and VM object 847 * @mem: ttm mem 848 * 849 * Fill in the page table entries for @bo_va. 850 * Returns 0 for success, -EINVAL for failure. 851 * 852 * Object have to be reserved and mutex must be locked! 853 */ 854 int amdgpu_vm_bo_update(struct amdgpu_device *adev, 855 struct amdgpu_bo_va *bo_va, 856 struct ttm_mem_reg *mem) 857 { 858 struct amdgpu_vm *vm = bo_va->vm; 859 struct amdgpu_bo_va_mapping *mapping; 860 uint32_t flags; 861 uint64_t addr; 862 int r; 863 864 if (mem) { 865 addr = mem->start << PAGE_SHIFT; 866 if (mem->mem_type != TTM_PL_TT) 867 addr += adev->vm_manager.vram_base_offset; 868 } else { 869 addr = 0; 870 } 871 872 flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem); 873 874 spin_lock(&vm->status_lock); 875 if (!list_empty(&bo_va->vm_status)) 876 list_splice_init(&bo_va->valids, &bo_va->invalids); 877 spin_unlock(&vm->status_lock); 878 879 list_for_each_entry(mapping, &bo_va->invalids, list) { 880 r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, addr, 881 flags, &bo_va->last_pt_update); 882 if (r) 883 return r; 884 } 885 886 spin_lock(&vm->status_lock); 887 list_splice_init(&bo_va->invalids, &bo_va->valids); 888 list_del_init(&bo_va->vm_status); 889 if (!mem) 890 list_add(&bo_va->vm_status, &vm->cleared); 891 spin_unlock(&vm->status_lock); 892 893 return 0; 894 } 895 896 /** 897 * amdgpu_vm_clear_freed - clear freed BOs in the PT 898 * 899 * @adev: amdgpu_device pointer 900 * @vm: requested vm 901 * 902 * Make sure all freed BOs are cleared in the PT. 903 * Returns 0 for success. 904 * 905 * PTs have to be reserved and mutex must be locked! 906 */ 907 int amdgpu_vm_clear_freed(struct amdgpu_device *adev, 908 struct amdgpu_vm *vm) 909 { 910 struct amdgpu_bo_va_mapping *mapping; 911 int r; 912 913 while (!list_empty(&vm->freed)) { 914 mapping = list_first_entry(&vm->freed, 915 struct amdgpu_bo_va_mapping, list); 916 list_del(&mapping->list); 917 918 r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, 0, 0, NULL); 919 kfree(mapping); 920 if (r) 921 return r; 922 923 } 924 return 0; 925 926 } 927 928 /** 929 * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT 930 * 931 * @adev: amdgpu_device pointer 932 * @vm: requested vm 933 * 934 * Make sure all invalidated BOs are cleared in the PT. 935 * Returns 0 for success. 936 * 937 * PTs have to be reserved and mutex must be locked! 938 */ 939 int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, 940 struct amdgpu_vm *vm, struct amdgpu_sync *sync) 941 { 942 struct amdgpu_bo_va *bo_va = NULL; 943 int r = 0; 944 945 spin_lock(&vm->status_lock); 946 while (!list_empty(&vm->invalidated)) { 947 bo_va = list_first_entry(&vm->invalidated, 948 struct amdgpu_bo_va, vm_status); 949 spin_unlock(&vm->status_lock); 950 951 r = amdgpu_vm_bo_update(adev, bo_va, NULL); 952 if (r) 953 return r; 954 955 spin_lock(&vm->status_lock); 956 } 957 spin_unlock(&vm->status_lock); 958 959 if (bo_va) 960 r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update); 961 962 return r; 963 } 964 965 /** 966 * amdgpu_vm_bo_add - add a bo to a specific vm 967 * 968 * @adev: amdgpu_device pointer 969 * @vm: requested vm 970 * @bo: amdgpu buffer object 971 * 972 * Add @bo into the requested vm (cayman+). 973 * Add @bo to the list of bos associated with the vm 974 * Returns newly added bo_va or NULL for failure 975 * 976 * Object has to be reserved! 977 */ 978 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, 979 struct amdgpu_vm *vm, 980 struct amdgpu_bo *bo) 981 { 982 struct amdgpu_bo_va *bo_va; 983 984 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL); 985 if (bo_va == NULL) { 986 return NULL; 987 } 988 bo_va->vm = vm; 989 bo_va->bo = bo; 990 bo_va->ref_count = 1; 991 INIT_LIST_HEAD(&bo_va->bo_list); 992 INIT_LIST_HEAD(&bo_va->valids); 993 INIT_LIST_HEAD(&bo_va->invalids); 994 INIT_LIST_HEAD(&bo_va->vm_status); 995 996 mutex_lock(&vm->mutex); 997 list_add_tail(&bo_va->bo_list, &bo->va); 998 mutex_unlock(&vm->mutex); 999 1000 return bo_va; 1001 } 1002 1003 /** 1004 * amdgpu_vm_bo_map - map bo inside a vm 1005 * 1006 * @adev: amdgpu_device pointer 1007 * @bo_va: bo_va to store the address 1008 * @saddr: where to map the BO 1009 * @offset: requested offset in the BO 1010 * @flags: attributes of pages (read/write/valid/etc.) 1011 * 1012 * Add a mapping of the BO at the specefied addr into the VM. 1013 * Returns 0 for success, error for failure. 1014 * 1015 * Object has to be reserved and gets unreserved by this function! 1016 */ 1017 int amdgpu_vm_bo_map(struct amdgpu_device *adev, 1018 struct amdgpu_bo_va *bo_va, 1019 uint64_t saddr, uint64_t offset, 1020 uint64_t size, uint32_t flags) 1021 { 1022 struct amdgpu_bo_va_mapping *mapping; 1023 struct amdgpu_vm *vm = bo_va->vm; 1024 struct interval_tree_node *it; 1025 unsigned last_pfn, pt_idx; 1026 uint64_t eaddr; 1027 int r; 1028 1029 /* validate the parameters */ 1030 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || 1031 size == 0 || size & AMDGPU_GPU_PAGE_MASK) { 1032 amdgpu_bo_unreserve(bo_va->bo); 1033 return -EINVAL; 1034 } 1035 1036 /* make sure object fit at this offset */ 1037 eaddr = saddr + size; 1038 if ((saddr >= eaddr) || (offset + size > amdgpu_bo_size(bo_va->bo))) { 1039 amdgpu_bo_unreserve(bo_va->bo); 1040 return -EINVAL; 1041 } 1042 1043 last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE; 1044 if (last_pfn > adev->vm_manager.max_pfn) { 1045 dev_err(adev->dev, "va above limit (0x%08X > 0x%08X)\n", 1046 last_pfn, adev->vm_manager.max_pfn); 1047 amdgpu_bo_unreserve(bo_va->bo); 1048 return -EINVAL; 1049 } 1050 1051 mutex_lock(&vm->mutex); 1052 1053 saddr /= AMDGPU_GPU_PAGE_SIZE; 1054 eaddr /= AMDGPU_GPU_PAGE_SIZE; 1055 1056 it = interval_tree_iter_first(&vm->va, saddr, eaddr - 1); 1057 if (it) { 1058 struct amdgpu_bo_va_mapping *tmp; 1059 tmp = container_of(it, struct amdgpu_bo_va_mapping, it); 1060 /* bo and tmp overlap, invalid addr */ 1061 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with " 1062 "0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr, 1063 tmp->it.start, tmp->it.last + 1); 1064 amdgpu_bo_unreserve(bo_va->bo); 1065 r = -EINVAL; 1066 goto error_unlock; 1067 } 1068 1069 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 1070 if (!mapping) { 1071 amdgpu_bo_unreserve(bo_va->bo); 1072 r = -ENOMEM; 1073 goto error_unlock; 1074 } 1075 1076 INIT_LIST_HEAD(&mapping->list); 1077 mapping->it.start = saddr; 1078 mapping->it.last = eaddr - 1; 1079 mapping->offset = offset; 1080 mapping->flags = flags; 1081 1082 list_add(&mapping->list, &bo_va->invalids); 1083 interval_tree_insert(&mapping->it, &vm->va); 1084 trace_amdgpu_vm_bo_map(bo_va, mapping); 1085 1086 /* Make sure the page tables are allocated */ 1087 saddr >>= amdgpu_vm_block_size; 1088 eaddr >>= amdgpu_vm_block_size; 1089 1090 BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev)); 1091 1092 if (eaddr > vm->max_pde_used) 1093 vm->max_pde_used = eaddr; 1094 1095 amdgpu_bo_unreserve(bo_va->bo); 1096 1097 /* walk over the address space and allocate the page tables */ 1098 for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) { 1099 struct amdgpu_bo *pt; 1100 1101 if (vm->page_tables[pt_idx].bo) 1102 continue; 1103 1104 /* drop mutex to allocate and clear page table */ 1105 mutex_unlock(&vm->mutex); 1106 1107 r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8, 1108 AMDGPU_GPU_PAGE_SIZE, true, 1109 AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &pt); 1110 if (r) 1111 goto error_free; 1112 1113 r = amdgpu_vm_clear_bo(adev, pt); 1114 if (r) { 1115 amdgpu_bo_unref(&pt); 1116 goto error_free; 1117 } 1118 1119 /* aquire mutex again */ 1120 mutex_lock(&vm->mutex); 1121 if (vm->page_tables[pt_idx].bo) { 1122 /* someone else allocated the pt in the meantime */ 1123 mutex_unlock(&vm->mutex); 1124 amdgpu_bo_unref(&pt); 1125 mutex_lock(&vm->mutex); 1126 continue; 1127 } 1128 1129 vm->page_tables[pt_idx].addr = 0; 1130 vm->page_tables[pt_idx].bo = pt; 1131 } 1132 1133 mutex_unlock(&vm->mutex); 1134 return 0; 1135 1136 error_free: 1137 mutex_lock(&vm->mutex); 1138 list_del(&mapping->list); 1139 interval_tree_remove(&mapping->it, &vm->va); 1140 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1141 kfree(mapping); 1142 1143 error_unlock: 1144 mutex_unlock(&vm->mutex); 1145 return r; 1146 } 1147 1148 /** 1149 * amdgpu_vm_bo_unmap - remove bo mapping from vm 1150 * 1151 * @adev: amdgpu_device pointer 1152 * @bo_va: bo_va to remove the address from 1153 * @saddr: where to the BO is mapped 1154 * 1155 * Remove a mapping of the BO at the specefied addr from the VM. 1156 * Returns 0 for success, error for failure. 1157 * 1158 * Object has to be reserved and gets unreserved by this function! 1159 */ 1160 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, 1161 struct amdgpu_bo_va *bo_va, 1162 uint64_t saddr) 1163 { 1164 struct amdgpu_bo_va_mapping *mapping; 1165 struct amdgpu_vm *vm = bo_va->vm; 1166 bool valid = true; 1167 1168 saddr /= AMDGPU_GPU_PAGE_SIZE; 1169 1170 list_for_each_entry(mapping, &bo_va->valids, list) { 1171 if (mapping->it.start == saddr) 1172 break; 1173 } 1174 1175 if (&mapping->list == &bo_va->valids) { 1176 valid = false; 1177 1178 list_for_each_entry(mapping, &bo_va->invalids, list) { 1179 if (mapping->it.start == saddr) 1180 break; 1181 } 1182 1183 if (&mapping->list == &bo_va->invalids) { 1184 amdgpu_bo_unreserve(bo_va->bo); 1185 return -ENOENT; 1186 } 1187 } 1188 1189 mutex_lock(&vm->mutex); 1190 list_del(&mapping->list); 1191 interval_tree_remove(&mapping->it, &vm->va); 1192 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1193 1194 if (valid) 1195 list_add(&mapping->list, &vm->freed); 1196 else 1197 kfree(mapping); 1198 mutex_unlock(&vm->mutex); 1199 amdgpu_bo_unreserve(bo_va->bo); 1200 1201 return 0; 1202 } 1203 1204 /** 1205 * amdgpu_vm_bo_rmv - remove a bo to a specific vm 1206 * 1207 * @adev: amdgpu_device pointer 1208 * @bo_va: requested bo_va 1209 * 1210 * Remove @bo_va->bo from the requested vm (cayman+). 1211 * 1212 * Object have to be reserved! 1213 */ 1214 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, 1215 struct amdgpu_bo_va *bo_va) 1216 { 1217 struct amdgpu_bo_va_mapping *mapping, *next; 1218 struct amdgpu_vm *vm = bo_va->vm; 1219 1220 list_del(&bo_va->bo_list); 1221 1222 mutex_lock(&vm->mutex); 1223 1224 spin_lock(&vm->status_lock); 1225 list_del(&bo_va->vm_status); 1226 spin_unlock(&vm->status_lock); 1227 1228 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 1229 list_del(&mapping->list); 1230 interval_tree_remove(&mapping->it, &vm->va); 1231 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1232 list_add(&mapping->list, &vm->freed); 1233 } 1234 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { 1235 list_del(&mapping->list); 1236 interval_tree_remove(&mapping->it, &vm->va); 1237 kfree(mapping); 1238 } 1239 1240 fence_put(bo_va->last_pt_update); 1241 kfree(bo_va); 1242 1243 mutex_unlock(&vm->mutex); 1244 } 1245 1246 /** 1247 * amdgpu_vm_bo_invalidate - mark the bo as invalid 1248 * 1249 * @adev: amdgpu_device pointer 1250 * @vm: requested vm 1251 * @bo: amdgpu buffer object 1252 * 1253 * Mark @bo as invalid (cayman+). 1254 */ 1255 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, 1256 struct amdgpu_bo *bo) 1257 { 1258 struct amdgpu_bo_va *bo_va; 1259 1260 list_for_each_entry(bo_va, &bo->va, bo_list) { 1261 spin_lock(&bo_va->vm->status_lock); 1262 if (list_empty(&bo_va->vm_status)) 1263 list_add(&bo_va->vm_status, &bo_va->vm->invalidated); 1264 spin_unlock(&bo_va->vm->status_lock); 1265 } 1266 } 1267 1268 /** 1269 * amdgpu_vm_init - initialize a vm instance 1270 * 1271 * @adev: amdgpu_device pointer 1272 * @vm: requested vm 1273 * 1274 * Init @vm fields (cayman+). 1275 */ 1276 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1277 { 1278 const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE, 1279 AMDGPU_VM_PTE_COUNT * 8); 1280 unsigned pd_size, pd_entries, pts_size; 1281 int i, r; 1282 1283 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 1284 vm->ids[i].id = 0; 1285 vm->ids[i].flushed_updates = NULL; 1286 vm->ids[i].last_id_use = NULL; 1287 } 1288 mutex_init(&vm->mutex); 1289 vm->va = RB_ROOT; 1290 spin_lock_init(&vm->status_lock); 1291 INIT_LIST_HEAD(&vm->invalidated); 1292 INIT_LIST_HEAD(&vm->cleared); 1293 INIT_LIST_HEAD(&vm->freed); 1294 1295 pd_size = amdgpu_vm_directory_size(adev); 1296 pd_entries = amdgpu_vm_num_pdes(adev); 1297 1298 /* allocate page table array */ 1299 pts_size = pd_entries * sizeof(struct amdgpu_vm_pt); 1300 vm->page_tables = kzalloc(pts_size, GFP_KERNEL); 1301 if (vm->page_tables == NULL) { 1302 DRM_ERROR("Cannot allocate memory for page table array\n"); 1303 return -ENOMEM; 1304 } 1305 1306 vm->page_directory_fence = NULL; 1307 1308 r = amdgpu_bo_create(adev, pd_size, align, true, 1309 AMDGPU_GEM_DOMAIN_VRAM, 0, 1310 NULL, &vm->page_directory); 1311 if (r) 1312 return r; 1313 1314 r = amdgpu_vm_clear_bo(adev, vm->page_directory); 1315 if (r) { 1316 amdgpu_bo_unref(&vm->page_directory); 1317 vm->page_directory = NULL; 1318 return r; 1319 } 1320 1321 return 0; 1322 } 1323 1324 /** 1325 * amdgpu_vm_fini - tear down a vm instance 1326 * 1327 * @adev: amdgpu_device pointer 1328 * @vm: requested vm 1329 * 1330 * Tear down @vm (cayman+). 1331 * Unbind the VM and remove all bos from the vm bo list 1332 */ 1333 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1334 { 1335 struct amdgpu_bo_va_mapping *mapping, *tmp; 1336 int i; 1337 1338 if (!RB_EMPTY_ROOT(&vm->va)) { 1339 dev_err(adev->dev, "still active bo inside vm\n"); 1340 } 1341 rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) { 1342 list_del(&mapping->list); 1343 interval_tree_remove(&mapping->it, &vm->va); 1344 kfree(mapping); 1345 } 1346 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { 1347 list_del(&mapping->list); 1348 kfree(mapping); 1349 } 1350 1351 for (i = 0; i < amdgpu_vm_num_pdes(adev); i++) 1352 amdgpu_bo_unref(&vm->page_tables[i].bo); 1353 kfree(vm->page_tables); 1354 1355 amdgpu_bo_unref(&vm->page_directory); 1356 fence_put(vm->page_directory_fence); 1357 1358 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 1359 fence_put(vm->ids[i].flushed_updates); 1360 amdgpu_fence_unref(&vm->ids[i].last_id_use); 1361 } 1362 1363 mutex_destroy(&vm->mutex); 1364 } 1365