1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <drm/drmP.h> 29 #include <drm/amdgpu_drm.h> 30 #include "amdgpu.h" 31 #include "amdgpu_trace.h" 32 33 /* 34 * GPUVM 35 * GPUVM is similar to the legacy gart on older asics, however 36 * rather than there being a single global gart table 37 * for the entire GPU, there are multiple VM page tables active 38 * at any given time. The VM page tables can contain a mix 39 * vram pages and system memory pages and system memory pages 40 * can be mapped as snooped (cached system pages) or unsnooped 41 * (uncached system pages). 42 * Each VM has an ID associated with it and there is a page table 43 * associated with each VMID. When execting a command buffer, 44 * the kernel tells the the ring what VMID to use for that command 45 * buffer. VMIDs are allocated dynamically as commands are submitted. 46 * The userspace drivers maintain their own address space and the kernel 47 * sets up their pages tables accordingly when they submit their 48 * command buffers and a VMID is assigned. 49 * Cayman/Trinity support up to 8 active VMs at any given time; 50 * SI supports 16. 51 */ 52 53 /** 54 * amdgpu_vm_num_pde - return the number of page directory entries 55 * 56 * @adev: amdgpu_device pointer 57 * 58 * Calculate the number of page directory entries (cayman+). 59 */ 60 static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev) 61 { 62 return adev->vm_manager.max_pfn >> amdgpu_vm_block_size; 63 } 64 65 /** 66 * amdgpu_vm_directory_size - returns the size of the page directory in bytes 67 * 68 * @adev: amdgpu_device pointer 69 * 70 * Calculate the size of the page directory in bytes (cayman+). 71 */ 72 static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev) 73 { 74 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8); 75 } 76 77 /** 78 * amdgpu_vm_get_bos - add the vm BOs to a validation list 79 * 80 * @vm: vm providing the BOs 81 * @head: head of validation list 82 * 83 * Add the page directory to the list of BOs to 84 * validate for command submission (cayman+). 85 */ 86 struct amdgpu_bo_list_entry *amdgpu_vm_get_bos(struct amdgpu_device *adev, 87 struct amdgpu_vm *vm, 88 struct list_head *head) 89 { 90 struct amdgpu_bo_list_entry *list; 91 unsigned i, idx; 92 93 mutex_lock(&vm->mutex); 94 list = drm_malloc_ab(vm->max_pde_used + 2, 95 sizeof(struct amdgpu_bo_list_entry)); 96 if (!list) { 97 mutex_unlock(&vm->mutex); 98 return NULL; 99 } 100 101 /* add the vm page table to the list */ 102 list[0].robj = vm->page_directory; 103 list[0].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM; 104 list[0].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM; 105 list[0].priority = 0; 106 list[0].tv.bo = &vm->page_directory->tbo; 107 list[0].tv.shared = true; 108 list_add(&list[0].tv.head, head); 109 110 for (i = 0, idx = 1; i <= vm->max_pde_used; i++) { 111 if (!vm->page_tables[i].bo) 112 continue; 113 114 list[idx].robj = vm->page_tables[i].bo; 115 list[idx].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM; 116 list[idx].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM; 117 list[idx].priority = 0; 118 list[idx].tv.bo = &list[idx].robj->tbo; 119 list[idx].tv.shared = true; 120 list_add(&list[idx++].tv.head, head); 121 } 122 mutex_unlock(&vm->mutex); 123 124 return list; 125 } 126 127 /** 128 * amdgpu_vm_grab_id - allocate the next free VMID 129 * 130 * @vm: vm to allocate id for 131 * @ring: ring we want to submit job to 132 * @sync: sync object where we add dependencies 133 * 134 * Allocate an id for the vm, adding fences to the sync obj as necessary. 135 * 136 * Global mutex must be locked! 137 */ 138 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, 139 struct amdgpu_sync *sync) 140 { 141 struct amdgpu_fence *best[AMDGPU_MAX_RINGS] = {}; 142 struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; 143 struct amdgpu_device *adev = ring->adev; 144 145 unsigned choices[2] = {}; 146 unsigned i; 147 148 /* check if the id is still valid */ 149 if (vm_id->id && vm_id->last_id_use && 150 vm_id->last_id_use == adev->vm_manager.active[vm_id->id]) 151 return 0; 152 153 /* we definately need to flush */ 154 vm_id->pd_gpu_addr = ~0ll; 155 156 /* skip over VMID 0, since it is the system VM */ 157 for (i = 1; i < adev->vm_manager.nvm; ++i) { 158 struct amdgpu_fence *fence = adev->vm_manager.active[i]; 159 160 if (fence == NULL) { 161 /* found a free one */ 162 vm_id->id = i; 163 trace_amdgpu_vm_grab_id(i, ring->idx); 164 return 0; 165 } 166 167 if (amdgpu_fence_is_earlier(fence, best[fence->ring->idx])) { 168 best[fence->ring->idx] = fence; 169 choices[fence->ring == ring ? 0 : 1] = i; 170 } 171 } 172 173 for (i = 0; i < 2; ++i) { 174 if (choices[i]) { 175 struct amdgpu_fence *fence; 176 177 fence = adev->vm_manager.active[choices[i]]; 178 vm_id->id = choices[i]; 179 180 trace_amdgpu_vm_grab_id(choices[i], ring->idx); 181 return amdgpu_sync_fence(ring->adev, sync, &fence->base); 182 } 183 } 184 185 /* should never happen */ 186 BUG(); 187 return -EINVAL; 188 } 189 190 /** 191 * amdgpu_vm_flush - hardware flush the vm 192 * 193 * @ring: ring to use for flush 194 * @vm: vm we want to flush 195 * @updates: last vm update that we waited for 196 * 197 * Flush the vm (cayman+). 198 * 199 * Global and local mutex must be locked! 200 */ 201 void amdgpu_vm_flush(struct amdgpu_ring *ring, 202 struct amdgpu_vm *vm, 203 struct fence *updates) 204 { 205 uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory); 206 struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; 207 struct fence *flushed_updates = vm_id->flushed_updates; 208 bool is_earlier = false; 209 210 if (flushed_updates && updates) { 211 BUG_ON(flushed_updates->context != updates->context); 212 is_earlier = (updates->seqno - flushed_updates->seqno <= 213 INT_MAX) ? true : false; 214 } 215 216 if (pd_addr != vm_id->pd_gpu_addr || !flushed_updates || 217 is_earlier) { 218 219 trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id->id); 220 if (is_earlier) { 221 vm_id->flushed_updates = fence_get(updates); 222 fence_put(flushed_updates); 223 } 224 if (!flushed_updates) 225 vm_id->flushed_updates = fence_get(updates); 226 vm_id->pd_gpu_addr = pd_addr; 227 amdgpu_ring_emit_vm_flush(ring, vm_id->id, vm_id->pd_gpu_addr); 228 } 229 } 230 231 /** 232 * amdgpu_vm_fence - remember fence for vm 233 * 234 * @adev: amdgpu_device pointer 235 * @vm: vm we want to fence 236 * @fence: fence to remember 237 * 238 * Fence the vm (cayman+). 239 * Set the fence used to protect page table and id. 240 * 241 * Global and local mutex must be locked! 242 */ 243 void amdgpu_vm_fence(struct amdgpu_device *adev, 244 struct amdgpu_vm *vm, 245 struct amdgpu_fence *fence) 246 { 247 unsigned ridx = fence->ring->idx; 248 unsigned vm_id = vm->ids[ridx].id; 249 250 amdgpu_fence_unref(&adev->vm_manager.active[vm_id]); 251 adev->vm_manager.active[vm_id] = amdgpu_fence_ref(fence); 252 253 amdgpu_fence_unref(&vm->ids[ridx].last_id_use); 254 vm->ids[ridx].last_id_use = amdgpu_fence_ref(fence); 255 } 256 257 /** 258 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo 259 * 260 * @vm: requested vm 261 * @bo: requested buffer object 262 * 263 * Find @bo inside the requested vm (cayman+). 264 * Search inside the @bos vm list for the requested vm 265 * Returns the found bo_va or NULL if none is found 266 * 267 * Object has to be reserved! 268 */ 269 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, 270 struct amdgpu_bo *bo) 271 { 272 struct amdgpu_bo_va *bo_va; 273 274 list_for_each_entry(bo_va, &bo->va, bo_list) { 275 if (bo_va->vm == vm) { 276 return bo_va; 277 } 278 } 279 return NULL; 280 } 281 282 /** 283 * amdgpu_vm_update_pages - helper to call the right asic function 284 * 285 * @adev: amdgpu_device pointer 286 * @ib: indirect buffer to fill with commands 287 * @pe: addr of the page entry 288 * @addr: dst addr to write into pe 289 * @count: number of page entries to update 290 * @incr: increase next addr by incr bytes 291 * @flags: hw access flags 292 * @gtt_flags: GTT hw access flags 293 * 294 * Traces the parameters and calls the right asic functions 295 * to setup the page table using the DMA. 296 */ 297 static void amdgpu_vm_update_pages(struct amdgpu_device *adev, 298 struct amdgpu_ib *ib, 299 uint64_t pe, uint64_t addr, 300 unsigned count, uint32_t incr, 301 uint32_t flags, uint32_t gtt_flags) 302 { 303 trace_amdgpu_vm_set_page(pe, addr, count, incr, flags); 304 305 if ((flags & AMDGPU_PTE_SYSTEM) && (flags == gtt_flags)) { 306 uint64_t src = adev->gart.table_addr + (addr >> 12) * 8; 307 amdgpu_vm_copy_pte(adev, ib, pe, src, count); 308 309 } else if ((flags & AMDGPU_PTE_SYSTEM) || (count < 3)) { 310 amdgpu_vm_write_pte(adev, ib, pe, addr, 311 count, incr, flags); 312 313 } else { 314 amdgpu_vm_set_pte_pde(adev, ib, pe, addr, 315 count, incr, flags); 316 } 317 } 318 319 int amdgpu_vm_free_job(struct amdgpu_job *job) 320 { 321 int i; 322 for (i = 0; i < job->num_ibs; i++) 323 amdgpu_ib_free(job->adev, &job->ibs[i]); 324 kfree(job->ibs); 325 return 0; 326 } 327 328 /** 329 * amdgpu_vm_clear_bo - initially clear the page dir/table 330 * 331 * @adev: amdgpu_device pointer 332 * @bo: bo to clear 333 */ 334 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, 335 struct amdgpu_bo *bo) 336 { 337 struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; 338 struct fence *fence = NULL; 339 struct amdgpu_ib *ib; 340 unsigned entries; 341 uint64_t addr; 342 int r; 343 344 r = amdgpu_bo_reserve(bo, false); 345 if (r) 346 return r; 347 348 r = reservation_object_reserve_shared(bo->tbo.resv); 349 if (r) 350 return r; 351 352 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 353 if (r) 354 goto error_unreserve; 355 356 addr = amdgpu_bo_gpu_offset(bo); 357 entries = amdgpu_bo_size(bo) / 8; 358 359 ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); 360 if (!ib) 361 goto error_unreserve; 362 363 r = amdgpu_ib_get(ring, NULL, entries * 2 + 64, ib); 364 if (r) 365 goto error_free; 366 367 ib->length_dw = 0; 368 369 amdgpu_vm_update_pages(adev, ib, addr, 0, entries, 0, 0, 0); 370 amdgpu_vm_pad_ib(adev, ib); 371 WARN_ON(ib->length_dw > 64); 372 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, 373 &amdgpu_vm_free_job, 374 AMDGPU_FENCE_OWNER_VM, 375 &fence); 376 if (!r) 377 amdgpu_bo_fence(bo, fence, true); 378 fence_put(fence); 379 if (amdgpu_enable_scheduler) { 380 amdgpu_bo_unreserve(bo); 381 return 0; 382 } 383 error_free: 384 amdgpu_ib_free(adev, ib); 385 kfree(ib); 386 387 error_unreserve: 388 amdgpu_bo_unreserve(bo); 389 return r; 390 } 391 392 /** 393 * amdgpu_vm_map_gart - get the physical address of a gart page 394 * 395 * @adev: amdgpu_device pointer 396 * @addr: the unmapped addr 397 * 398 * Look up the physical address of the page that the pte resolves 399 * to (cayman+). 400 * Returns the physical address of the page. 401 */ 402 uint64_t amdgpu_vm_map_gart(struct amdgpu_device *adev, uint64_t addr) 403 { 404 uint64_t result; 405 406 /* page table offset */ 407 result = adev->gart.pages_addr[addr >> PAGE_SHIFT]; 408 409 /* in case cpu page size != gpu page size*/ 410 result |= addr & (~PAGE_MASK); 411 412 return result; 413 } 414 415 /** 416 * amdgpu_vm_update_pdes - make sure that page directory is valid 417 * 418 * @adev: amdgpu_device pointer 419 * @vm: requested vm 420 * @start: start of GPU address range 421 * @end: end of GPU address range 422 * 423 * Allocates new page tables if necessary 424 * and updates the page directory (cayman+). 425 * Returns 0 for success, error for failure. 426 * 427 * Global and local mutex must be locked! 428 */ 429 int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, 430 struct amdgpu_vm *vm) 431 { 432 struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; 433 struct amdgpu_bo *pd = vm->page_directory; 434 uint64_t pd_addr = amdgpu_bo_gpu_offset(pd); 435 uint32_t incr = AMDGPU_VM_PTE_COUNT * 8; 436 uint64_t last_pde = ~0, last_pt = ~0; 437 unsigned count = 0, pt_idx, ndw; 438 struct amdgpu_ib *ib; 439 struct fence *fence = NULL; 440 441 int r; 442 443 /* padding, etc. */ 444 ndw = 64; 445 446 /* assume the worst case */ 447 ndw += vm->max_pde_used * 6; 448 449 /* update too big for an IB */ 450 if (ndw > 0xfffff) 451 return -ENOMEM; 452 453 ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); 454 if (!ib) 455 return -ENOMEM; 456 457 r = amdgpu_ib_get(ring, NULL, ndw * 4, ib); 458 if (r) 459 return r; 460 ib->length_dw = 0; 461 462 /* walk over the address space and update the page directory */ 463 for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) { 464 struct amdgpu_bo *bo = vm->page_tables[pt_idx].bo; 465 uint64_t pde, pt; 466 467 if (bo == NULL) 468 continue; 469 470 pt = amdgpu_bo_gpu_offset(bo); 471 if (vm->page_tables[pt_idx].addr == pt) 472 continue; 473 vm->page_tables[pt_idx].addr = pt; 474 475 pde = pd_addr + pt_idx * 8; 476 if (((last_pde + 8 * count) != pde) || 477 ((last_pt + incr * count) != pt)) { 478 479 if (count) { 480 amdgpu_vm_update_pages(adev, ib, last_pde, 481 last_pt, count, incr, 482 AMDGPU_PTE_VALID, 0); 483 } 484 485 count = 1; 486 last_pde = pde; 487 last_pt = pt; 488 } else { 489 ++count; 490 } 491 } 492 493 if (count) 494 amdgpu_vm_update_pages(adev, ib, last_pde, last_pt, count, 495 incr, AMDGPU_PTE_VALID, 0); 496 497 if (ib->length_dw != 0) { 498 amdgpu_vm_pad_ib(adev, ib); 499 amdgpu_sync_resv(adev, &ib->sync, pd->tbo.resv, AMDGPU_FENCE_OWNER_VM); 500 WARN_ON(ib->length_dw > ndw); 501 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, 502 &amdgpu_vm_free_job, 503 AMDGPU_FENCE_OWNER_VM, 504 &fence); 505 if (r) 506 goto error_free; 507 508 amdgpu_bo_fence(pd, fence, true); 509 fence_put(vm->page_directory_fence); 510 vm->page_directory_fence = fence_get(fence); 511 fence_put(fence); 512 } 513 514 if (!amdgpu_enable_scheduler || ib->length_dw == 0) { 515 amdgpu_ib_free(adev, ib); 516 kfree(ib); 517 } 518 519 return 0; 520 521 error_free: 522 amdgpu_ib_free(adev, ib); 523 kfree(ib); 524 return r; 525 } 526 527 /** 528 * amdgpu_vm_frag_ptes - add fragment information to PTEs 529 * 530 * @adev: amdgpu_device pointer 531 * @ib: IB for the update 532 * @pe_start: first PTE to handle 533 * @pe_end: last PTE to handle 534 * @addr: addr those PTEs should point to 535 * @flags: hw mapping flags 536 * @gtt_flags: GTT hw mapping flags 537 * 538 * Global and local mutex must be locked! 539 */ 540 static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev, 541 struct amdgpu_ib *ib, 542 uint64_t pe_start, uint64_t pe_end, 543 uint64_t addr, uint32_t flags, 544 uint32_t gtt_flags) 545 { 546 /** 547 * The MC L1 TLB supports variable sized pages, based on a fragment 548 * field in the PTE. When this field is set to a non-zero value, page 549 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE 550 * flags are considered valid for all PTEs within the fragment range 551 * and corresponding mappings are assumed to be physically contiguous. 552 * 553 * The L1 TLB can store a single PTE for the whole fragment, 554 * significantly increasing the space available for translation 555 * caching. This leads to large improvements in throughput when the 556 * TLB is under pressure. 557 * 558 * The L2 TLB distributes small and large fragments into two 559 * asymmetric partitions. The large fragment cache is significantly 560 * larger. Thus, we try to use large fragments wherever possible. 561 * Userspace can support this by aligning virtual base address and 562 * allocation size to the fragment size. 563 */ 564 565 /* SI and newer are optimized for 64KB */ 566 uint64_t frag_flags = AMDGPU_PTE_FRAG_64KB; 567 uint64_t frag_align = 0x80; 568 569 uint64_t frag_start = ALIGN(pe_start, frag_align); 570 uint64_t frag_end = pe_end & ~(frag_align - 1); 571 572 unsigned count; 573 574 /* system pages are non continuously */ 575 if ((flags & AMDGPU_PTE_SYSTEM) || !(flags & AMDGPU_PTE_VALID) || 576 (frag_start >= frag_end)) { 577 578 count = (pe_end - pe_start) / 8; 579 amdgpu_vm_update_pages(adev, ib, pe_start, addr, count, 580 AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); 581 return; 582 } 583 584 /* handle the 4K area at the beginning */ 585 if (pe_start != frag_start) { 586 count = (frag_start - pe_start) / 8; 587 amdgpu_vm_update_pages(adev, ib, pe_start, addr, count, 588 AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); 589 addr += AMDGPU_GPU_PAGE_SIZE * count; 590 } 591 592 /* handle the area in the middle */ 593 count = (frag_end - frag_start) / 8; 594 amdgpu_vm_update_pages(adev, ib, frag_start, addr, count, 595 AMDGPU_GPU_PAGE_SIZE, flags | frag_flags, 596 gtt_flags); 597 598 /* handle the 4K area at the end */ 599 if (frag_end != pe_end) { 600 addr += AMDGPU_GPU_PAGE_SIZE * count; 601 count = (pe_end - frag_end) / 8; 602 amdgpu_vm_update_pages(adev, ib, frag_end, addr, count, 603 AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); 604 } 605 } 606 607 /** 608 * amdgpu_vm_update_ptes - make sure that page tables are valid 609 * 610 * @adev: amdgpu_device pointer 611 * @vm: requested vm 612 * @start: start of GPU address range 613 * @end: end of GPU address range 614 * @dst: destination address to map to 615 * @flags: mapping flags 616 * 617 * Update the page tables in the range @start - @end (cayman+). 618 * 619 * Global and local mutex must be locked! 620 */ 621 static int amdgpu_vm_update_ptes(struct amdgpu_device *adev, 622 struct amdgpu_vm *vm, 623 struct amdgpu_ib *ib, 624 uint64_t start, uint64_t end, 625 uint64_t dst, uint32_t flags, 626 uint32_t gtt_flags) 627 { 628 uint64_t mask = AMDGPU_VM_PTE_COUNT - 1; 629 uint64_t last_pte = ~0, last_dst = ~0; 630 void *owner = AMDGPU_FENCE_OWNER_VM; 631 unsigned count = 0; 632 uint64_t addr; 633 634 /* sync to everything on unmapping */ 635 if (!(flags & AMDGPU_PTE_VALID)) 636 owner = AMDGPU_FENCE_OWNER_UNDEFINED; 637 638 /* walk over the address space and update the page tables */ 639 for (addr = start; addr < end; ) { 640 uint64_t pt_idx = addr >> amdgpu_vm_block_size; 641 struct amdgpu_bo *pt = vm->page_tables[pt_idx].bo; 642 unsigned nptes; 643 uint64_t pte; 644 int r; 645 646 amdgpu_sync_resv(adev, &ib->sync, pt->tbo.resv, owner); 647 r = reservation_object_reserve_shared(pt->tbo.resv); 648 if (r) 649 return r; 650 651 if ((addr & ~mask) == (end & ~mask)) 652 nptes = end - addr; 653 else 654 nptes = AMDGPU_VM_PTE_COUNT - (addr & mask); 655 656 pte = amdgpu_bo_gpu_offset(pt); 657 pte += (addr & mask) * 8; 658 659 if ((last_pte + 8 * count) != pte) { 660 661 if (count) { 662 amdgpu_vm_frag_ptes(adev, ib, last_pte, 663 last_pte + 8 * count, 664 last_dst, flags, 665 gtt_flags); 666 } 667 668 count = nptes; 669 last_pte = pte; 670 last_dst = dst; 671 } else { 672 count += nptes; 673 } 674 675 addr += nptes; 676 dst += nptes * AMDGPU_GPU_PAGE_SIZE; 677 } 678 679 if (count) { 680 amdgpu_vm_frag_ptes(adev, ib, last_pte, 681 last_pte + 8 * count, 682 last_dst, flags, gtt_flags); 683 } 684 685 return 0; 686 } 687 688 /** 689 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table 690 * 691 * @adev: amdgpu_device pointer 692 * @vm: requested vm 693 * @mapping: mapped range and flags to use for the update 694 * @addr: addr to set the area to 695 * @gtt_flags: flags as they are used for GTT 696 * @fence: optional resulting fence 697 * 698 * Fill in the page table entries for @mapping. 699 * Returns 0 for success, -EINVAL for failure. 700 * 701 * Object have to be reserved and mutex must be locked! 702 */ 703 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, 704 struct amdgpu_vm *vm, 705 struct amdgpu_bo_va_mapping *mapping, 706 uint64_t addr, uint32_t gtt_flags, 707 struct fence **fence) 708 { 709 struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; 710 unsigned nptes, ncmds, ndw; 711 uint32_t flags = gtt_flags; 712 struct amdgpu_ib *ib; 713 struct fence *f = NULL; 714 int r; 715 716 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here 717 * but in case of something, we filter the flags in first place 718 */ 719 if (!(mapping->flags & AMDGPU_PTE_READABLE)) 720 flags &= ~AMDGPU_PTE_READABLE; 721 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) 722 flags &= ~AMDGPU_PTE_WRITEABLE; 723 724 trace_amdgpu_vm_bo_update(mapping); 725 726 nptes = mapping->it.last - mapping->it.start + 1; 727 728 /* 729 * reserve space for one command every (1 << BLOCK_SIZE) 730 * entries or 2k dwords (whatever is smaller) 731 */ 732 ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1; 733 734 /* padding, etc. */ 735 ndw = 64; 736 737 if ((flags & AMDGPU_PTE_SYSTEM) && (flags == gtt_flags)) { 738 /* only copy commands needed */ 739 ndw += ncmds * 7; 740 741 } else if (flags & AMDGPU_PTE_SYSTEM) { 742 /* header for write data commands */ 743 ndw += ncmds * 4; 744 745 /* body of write data command */ 746 ndw += nptes * 2; 747 748 } else { 749 /* set page commands needed */ 750 ndw += ncmds * 10; 751 752 /* two extra commands for begin/end of fragment */ 753 ndw += 2 * 10; 754 } 755 756 /* update too big for an IB */ 757 if (ndw > 0xfffff) 758 return -ENOMEM; 759 760 ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); 761 if (!ib) 762 return -ENOMEM; 763 764 r = amdgpu_ib_get(ring, NULL, ndw * 4, ib); 765 if (r) { 766 kfree(ib); 767 return r; 768 } 769 770 ib->length_dw = 0; 771 772 r = amdgpu_vm_update_ptes(adev, vm, ib, mapping->it.start, 773 mapping->it.last + 1, addr + mapping->offset, 774 flags, gtt_flags); 775 776 if (r) { 777 amdgpu_ib_free(adev, ib); 778 kfree(ib); 779 return r; 780 } 781 782 amdgpu_vm_pad_ib(adev, ib); 783 WARN_ON(ib->length_dw > ndw); 784 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, 785 &amdgpu_vm_free_job, 786 AMDGPU_FENCE_OWNER_VM, 787 &f); 788 if (r) 789 goto error_free; 790 791 amdgpu_bo_fence(vm->page_directory, f, true); 792 if (fence) { 793 fence_put(*fence); 794 *fence = fence_get(f); 795 } 796 fence_put(f); 797 if (!amdgpu_enable_scheduler) { 798 amdgpu_ib_free(adev, ib); 799 kfree(ib); 800 } 801 return 0; 802 803 error_free: 804 amdgpu_ib_free(adev, ib); 805 kfree(ib); 806 return r; 807 } 808 809 /** 810 * amdgpu_vm_bo_update - update all BO mappings in the vm page table 811 * 812 * @adev: amdgpu_device pointer 813 * @bo_va: requested BO and VM object 814 * @mem: ttm mem 815 * 816 * Fill in the page table entries for @bo_va. 817 * Returns 0 for success, -EINVAL for failure. 818 * 819 * Object have to be reserved and mutex must be locked! 820 */ 821 int amdgpu_vm_bo_update(struct amdgpu_device *adev, 822 struct amdgpu_bo_va *bo_va, 823 struct ttm_mem_reg *mem) 824 { 825 struct amdgpu_vm *vm = bo_va->vm; 826 struct amdgpu_bo_va_mapping *mapping; 827 uint32_t flags; 828 uint64_t addr; 829 int r; 830 831 if (mem) { 832 addr = (u64)mem->start << PAGE_SHIFT; 833 if (mem->mem_type != TTM_PL_TT) 834 addr += adev->vm_manager.vram_base_offset; 835 } else { 836 addr = 0; 837 } 838 839 flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem); 840 841 spin_lock(&vm->status_lock); 842 if (!list_empty(&bo_va->vm_status)) 843 list_splice_init(&bo_va->valids, &bo_va->invalids); 844 spin_unlock(&vm->status_lock); 845 846 list_for_each_entry(mapping, &bo_va->invalids, list) { 847 r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, addr, 848 flags, &bo_va->last_pt_update); 849 if (r) 850 return r; 851 } 852 853 spin_lock(&vm->status_lock); 854 list_splice_init(&bo_va->invalids, &bo_va->valids); 855 list_del_init(&bo_va->vm_status); 856 if (!mem) 857 list_add(&bo_va->vm_status, &vm->cleared); 858 spin_unlock(&vm->status_lock); 859 860 return 0; 861 } 862 863 /** 864 * amdgpu_vm_clear_freed - clear freed BOs in the PT 865 * 866 * @adev: amdgpu_device pointer 867 * @vm: requested vm 868 * 869 * Make sure all freed BOs are cleared in the PT. 870 * Returns 0 for success. 871 * 872 * PTs have to be reserved and mutex must be locked! 873 */ 874 int amdgpu_vm_clear_freed(struct amdgpu_device *adev, 875 struct amdgpu_vm *vm) 876 { 877 struct amdgpu_bo_va_mapping *mapping; 878 int r; 879 880 while (!list_empty(&vm->freed)) { 881 mapping = list_first_entry(&vm->freed, 882 struct amdgpu_bo_va_mapping, list); 883 list_del(&mapping->list); 884 885 r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, 0, 0, NULL); 886 kfree(mapping); 887 if (r) 888 return r; 889 890 } 891 return 0; 892 893 } 894 895 /** 896 * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT 897 * 898 * @adev: amdgpu_device pointer 899 * @vm: requested vm 900 * 901 * Make sure all invalidated BOs are cleared in the PT. 902 * Returns 0 for success. 903 * 904 * PTs have to be reserved and mutex must be locked! 905 */ 906 int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, 907 struct amdgpu_vm *vm, struct amdgpu_sync *sync) 908 { 909 struct amdgpu_bo_va *bo_va = NULL; 910 int r = 0; 911 912 spin_lock(&vm->status_lock); 913 while (!list_empty(&vm->invalidated)) { 914 bo_va = list_first_entry(&vm->invalidated, 915 struct amdgpu_bo_va, vm_status); 916 spin_unlock(&vm->status_lock); 917 918 r = amdgpu_vm_bo_update(adev, bo_va, NULL); 919 if (r) 920 return r; 921 922 spin_lock(&vm->status_lock); 923 } 924 spin_unlock(&vm->status_lock); 925 926 if (bo_va) 927 r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update); 928 929 return r; 930 } 931 932 /** 933 * amdgpu_vm_bo_add - add a bo to a specific vm 934 * 935 * @adev: amdgpu_device pointer 936 * @vm: requested vm 937 * @bo: amdgpu buffer object 938 * 939 * Add @bo into the requested vm (cayman+). 940 * Add @bo to the list of bos associated with the vm 941 * Returns newly added bo_va or NULL for failure 942 * 943 * Object has to be reserved! 944 */ 945 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, 946 struct amdgpu_vm *vm, 947 struct amdgpu_bo *bo) 948 { 949 struct amdgpu_bo_va *bo_va; 950 951 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL); 952 if (bo_va == NULL) { 953 return NULL; 954 } 955 bo_va->vm = vm; 956 bo_va->bo = bo; 957 bo_va->ref_count = 1; 958 INIT_LIST_HEAD(&bo_va->bo_list); 959 INIT_LIST_HEAD(&bo_va->valids); 960 INIT_LIST_HEAD(&bo_va->invalids); 961 INIT_LIST_HEAD(&bo_va->vm_status); 962 963 mutex_lock(&vm->mutex); 964 list_add_tail(&bo_va->bo_list, &bo->va); 965 mutex_unlock(&vm->mutex); 966 967 return bo_va; 968 } 969 970 /** 971 * amdgpu_vm_bo_map - map bo inside a vm 972 * 973 * @adev: amdgpu_device pointer 974 * @bo_va: bo_va to store the address 975 * @saddr: where to map the BO 976 * @offset: requested offset in the BO 977 * @flags: attributes of pages (read/write/valid/etc.) 978 * 979 * Add a mapping of the BO at the specefied addr into the VM. 980 * Returns 0 for success, error for failure. 981 * 982 * Object has to be reserved and gets unreserved by this function! 983 */ 984 int amdgpu_vm_bo_map(struct amdgpu_device *adev, 985 struct amdgpu_bo_va *bo_va, 986 uint64_t saddr, uint64_t offset, 987 uint64_t size, uint32_t flags) 988 { 989 struct amdgpu_bo_va_mapping *mapping; 990 struct amdgpu_vm *vm = bo_va->vm; 991 struct interval_tree_node *it; 992 unsigned last_pfn, pt_idx; 993 uint64_t eaddr; 994 int r; 995 996 /* validate the parameters */ 997 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || 998 size == 0 || size & AMDGPU_GPU_PAGE_MASK) { 999 amdgpu_bo_unreserve(bo_va->bo); 1000 return -EINVAL; 1001 } 1002 1003 /* make sure object fit at this offset */ 1004 eaddr = saddr + size; 1005 if ((saddr >= eaddr) || (offset + size > amdgpu_bo_size(bo_va->bo))) { 1006 amdgpu_bo_unreserve(bo_va->bo); 1007 return -EINVAL; 1008 } 1009 1010 last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE; 1011 if (last_pfn > adev->vm_manager.max_pfn) { 1012 dev_err(adev->dev, "va above limit (0x%08X > 0x%08X)\n", 1013 last_pfn, adev->vm_manager.max_pfn); 1014 amdgpu_bo_unreserve(bo_va->bo); 1015 return -EINVAL; 1016 } 1017 1018 mutex_lock(&vm->mutex); 1019 1020 saddr /= AMDGPU_GPU_PAGE_SIZE; 1021 eaddr /= AMDGPU_GPU_PAGE_SIZE; 1022 1023 it = interval_tree_iter_first(&vm->va, saddr, eaddr - 1); 1024 if (it) { 1025 struct amdgpu_bo_va_mapping *tmp; 1026 tmp = container_of(it, struct amdgpu_bo_va_mapping, it); 1027 /* bo and tmp overlap, invalid addr */ 1028 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with " 1029 "0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr, 1030 tmp->it.start, tmp->it.last + 1); 1031 amdgpu_bo_unreserve(bo_va->bo); 1032 r = -EINVAL; 1033 goto error_unlock; 1034 } 1035 1036 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 1037 if (!mapping) { 1038 amdgpu_bo_unreserve(bo_va->bo); 1039 r = -ENOMEM; 1040 goto error_unlock; 1041 } 1042 1043 INIT_LIST_HEAD(&mapping->list); 1044 mapping->it.start = saddr; 1045 mapping->it.last = eaddr - 1; 1046 mapping->offset = offset; 1047 mapping->flags = flags; 1048 1049 list_add(&mapping->list, &bo_va->invalids); 1050 interval_tree_insert(&mapping->it, &vm->va); 1051 trace_amdgpu_vm_bo_map(bo_va, mapping); 1052 1053 /* Make sure the page tables are allocated */ 1054 saddr >>= amdgpu_vm_block_size; 1055 eaddr >>= amdgpu_vm_block_size; 1056 1057 BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev)); 1058 1059 if (eaddr > vm->max_pde_used) 1060 vm->max_pde_used = eaddr; 1061 1062 amdgpu_bo_unreserve(bo_va->bo); 1063 1064 /* walk over the address space and allocate the page tables */ 1065 for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) { 1066 struct reservation_object *resv = vm->page_directory->tbo.resv; 1067 struct amdgpu_bo *pt; 1068 1069 if (vm->page_tables[pt_idx].bo) 1070 continue; 1071 1072 /* drop mutex to allocate and clear page table */ 1073 mutex_unlock(&vm->mutex); 1074 1075 ww_mutex_lock(&resv->lock, NULL); 1076 r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8, 1077 AMDGPU_GPU_PAGE_SIZE, true, 1078 AMDGPU_GEM_DOMAIN_VRAM, 1079 AMDGPU_GEM_CREATE_NO_CPU_ACCESS, 1080 NULL, resv, &pt); 1081 ww_mutex_unlock(&resv->lock); 1082 if (r) 1083 goto error_free; 1084 1085 r = amdgpu_vm_clear_bo(adev, pt); 1086 if (r) { 1087 amdgpu_bo_unref(&pt); 1088 goto error_free; 1089 } 1090 1091 /* aquire mutex again */ 1092 mutex_lock(&vm->mutex); 1093 if (vm->page_tables[pt_idx].bo) { 1094 /* someone else allocated the pt in the meantime */ 1095 mutex_unlock(&vm->mutex); 1096 amdgpu_bo_unref(&pt); 1097 mutex_lock(&vm->mutex); 1098 continue; 1099 } 1100 1101 vm->page_tables[pt_idx].addr = 0; 1102 vm->page_tables[pt_idx].bo = pt; 1103 } 1104 1105 mutex_unlock(&vm->mutex); 1106 return 0; 1107 1108 error_free: 1109 mutex_lock(&vm->mutex); 1110 list_del(&mapping->list); 1111 interval_tree_remove(&mapping->it, &vm->va); 1112 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1113 kfree(mapping); 1114 1115 error_unlock: 1116 mutex_unlock(&vm->mutex); 1117 return r; 1118 } 1119 1120 /** 1121 * amdgpu_vm_bo_unmap - remove bo mapping from vm 1122 * 1123 * @adev: amdgpu_device pointer 1124 * @bo_va: bo_va to remove the address from 1125 * @saddr: where to the BO is mapped 1126 * 1127 * Remove a mapping of the BO at the specefied addr from the VM. 1128 * Returns 0 for success, error for failure. 1129 * 1130 * Object has to be reserved and gets unreserved by this function! 1131 */ 1132 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, 1133 struct amdgpu_bo_va *bo_va, 1134 uint64_t saddr) 1135 { 1136 struct amdgpu_bo_va_mapping *mapping; 1137 struct amdgpu_vm *vm = bo_va->vm; 1138 bool valid = true; 1139 1140 saddr /= AMDGPU_GPU_PAGE_SIZE; 1141 1142 list_for_each_entry(mapping, &bo_va->valids, list) { 1143 if (mapping->it.start == saddr) 1144 break; 1145 } 1146 1147 if (&mapping->list == &bo_va->valids) { 1148 valid = false; 1149 1150 list_for_each_entry(mapping, &bo_va->invalids, list) { 1151 if (mapping->it.start == saddr) 1152 break; 1153 } 1154 1155 if (&mapping->list == &bo_va->invalids) { 1156 amdgpu_bo_unreserve(bo_va->bo); 1157 return -ENOENT; 1158 } 1159 } 1160 1161 mutex_lock(&vm->mutex); 1162 list_del(&mapping->list); 1163 interval_tree_remove(&mapping->it, &vm->va); 1164 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1165 1166 if (valid) 1167 list_add(&mapping->list, &vm->freed); 1168 else 1169 kfree(mapping); 1170 mutex_unlock(&vm->mutex); 1171 amdgpu_bo_unreserve(bo_va->bo); 1172 1173 return 0; 1174 } 1175 1176 /** 1177 * amdgpu_vm_bo_rmv - remove a bo to a specific vm 1178 * 1179 * @adev: amdgpu_device pointer 1180 * @bo_va: requested bo_va 1181 * 1182 * Remove @bo_va->bo from the requested vm (cayman+). 1183 * 1184 * Object have to be reserved! 1185 */ 1186 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, 1187 struct amdgpu_bo_va *bo_va) 1188 { 1189 struct amdgpu_bo_va_mapping *mapping, *next; 1190 struct amdgpu_vm *vm = bo_va->vm; 1191 1192 list_del(&bo_va->bo_list); 1193 1194 mutex_lock(&vm->mutex); 1195 1196 spin_lock(&vm->status_lock); 1197 list_del(&bo_va->vm_status); 1198 spin_unlock(&vm->status_lock); 1199 1200 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 1201 list_del(&mapping->list); 1202 interval_tree_remove(&mapping->it, &vm->va); 1203 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1204 list_add(&mapping->list, &vm->freed); 1205 } 1206 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { 1207 list_del(&mapping->list); 1208 interval_tree_remove(&mapping->it, &vm->va); 1209 kfree(mapping); 1210 } 1211 1212 fence_put(bo_va->last_pt_update); 1213 kfree(bo_va); 1214 1215 mutex_unlock(&vm->mutex); 1216 } 1217 1218 /** 1219 * amdgpu_vm_bo_invalidate - mark the bo as invalid 1220 * 1221 * @adev: amdgpu_device pointer 1222 * @vm: requested vm 1223 * @bo: amdgpu buffer object 1224 * 1225 * Mark @bo as invalid (cayman+). 1226 */ 1227 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, 1228 struct amdgpu_bo *bo) 1229 { 1230 struct amdgpu_bo_va *bo_va; 1231 1232 list_for_each_entry(bo_va, &bo->va, bo_list) { 1233 spin_lock(&bo_va->vm->status_lock); 1234 if (list_empty(&bo_va->vm_status)) 1235 list_add(&bo_va->vm_status, &bo_va->vm->invalidated); 1236 spin_unlock(&bo_va->vm->status_lock); 1237 } 1238 } 1239 1240 /** 1241 * amdgpu_vm_init - initialize a vm instance 1242 * 1243 * @adev: amdgpu_device pointer 1244 * @vm: requested vm 1245 * 1246 * Init @vm fields (cayman+). 1247 */ 1248 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1249 { 1250 const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE, 1251 AMDGPU_VM_PTE_COUNT * 8); 1252 unsigned pd_size, pd_entries, pts_size; 1253 int i, r; 1254 1255 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 1256 vm->ids[i].id = 0; 1257 vm->ids[i].flushed_updates = NULL; 1258 vm->ids[i].last_id_use = NULL; 1259 } 1260 mutex_init(&vm->mutex); 1261 vm->va = RB_ROOT; 1262 spin_lock_init(&vm->status_lock); 1263 INIT_LIST_HEAD(&vm->invalidated); 1264 INIT_LIST_HEAD(&vm->cleared); 1265 INIT_LIST_HEAD(&vm->freed); 1266 1267 pd_size = amdgpu_vm_directory_size(adev); 1268 pd_entries = amdgpu_vm_num_pdes(adev); 1269 1270 /* allocate page table array */ 1271 pts_size = pd_entries * sizeof(struct amdgpu_vm_pt); 1272 vm->page_tables = kzalloc(pts_size, GFP_KERNEL); 1273 if (vm->page_tables == NULL) { 1274 DRM_ERROR("Cannot allocate memory for page table array\n"); 1275 return -ENOMEM; 1276 } 1277 1278 vm->page_directory_fence = NULL; 1279 1280 r = amdgpu_bo_create(adev, pd_size, align, true, 1281 AMDGPU_GEM_DOMAIN_VRAM, 1282 AMDGPU_GEM_CREATE_NO_CPU_ACCESS, 1283 NULL, NULL, &vm->page_directory); 1284 if (r) 1285 return r; 1286 1287 r = amdgpu_vm_clear_bo(adev, vm->page_directory); 1288 if (r) { 1289 amdgpu_bo_unref(&vm->page_directory); 1290 vm->page_directory = NULL; 1291 return r; 1292 } 1293 1294 return 0; 1295 } 1296 1297 /** 1298 * amdgpu_vm_fini - tear down a vm instance 1299 * 1300 * @adev: amdgpu_device pointer 1301 * @vm: requested vm 1302 * 1303 * Tear down @vm (cayman+). 1304 * Unbind the VM and remove all bos from the vm bo list 1305 */ 1306 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1307 { 1308 struct amdgpu_bo_va_mapping *mapping, *tmp; 1309 int i; 1310 1311 if (!RB_EMPTY_ROOT(&vm->va)) { 1312 dev_err(adev->dev, "still active bo inside vm\n"); 1313 } 1314 rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) { 1315 list_del(&mapping->list); 1316 interval_tree_remove(&mapping->it, &vm->va); 1317 kfree(mapping); 1318 } 1319 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { 1320 list_del(&mapping->list); 1321 kfree(mapping); 1322 } 1323 1324 for (i = 0; i < amdgpu_vm_num_pdes(adev); i++) 1325 amdgpu_bo_unref(&vm->page_tables[i].bo); 1326 kfree(vm->page_tables); 1327 1328 amdgpu_bo_unref(&vm->page_directory); 1329 fence_put(vm->page_directory_fence); 1330 1331 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 1332 fence_put(vm->ids[i].flushed_updates); 1333 amdgpu_fence_unref(&vm->ids[i].last_id_use); 1334 } 1335 1336 mutex_destroy(&vm->mutex); 1337 } 1338