1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <drm/drmP.h> 29 #include <drm/amdgpu_drm.h> 30 #include "amdgpu.h" 31 #include "amdgpu_trace.h" 32 33 /* 34 * GPUVM 35 * GPUVM is similar to the legacy gart on older asics, however 36 * rather than there being a single global gart table 37 * for the entire GPU, there are multiple VM page tables active 38 * at any given time. The VM page tables can contain a mix 39 * vram pages and system memory pages and system memory pages 40 * can be mapped as snooped (cached system pages) or unsnooped 41 * (uncached system pages). 42 * Each VM has an ID associated with it and there is a page table 43 * associated with each VMID. When execting a command buffer, 44 * the kernel tells the the ring what VMID to use for that command 45 * buffer. VMIDs are allocated dynamically as commands are submitted. 46 * The userspace drivers maintain their own address space and the kernel 47 * sets up their pages tables accordingly when they submit their 48 * command buffers and a VMID is assigned. 49 * Cayman/Trinity support up to 8 active VMs at any given time; 50 * SI supports 16. 51 */ 52 53 /** 54 * amdgpu_vm_num_pde - return the number of page directory entries 55 * 56 * @adev: amdgpu_device pointer 57 * 58 * Calculate the number of page directory entries (cayman+). 59 */ 60 static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev) 61 { 62 return adev->vm_manager.max_pfn >> amdgpu_vm_block_size; 63 } 64 65 /** 66 * amdgpu_vm_directory_size - returns the size of the page directory in bytes 67 * 68 * @adev: amdgpu_device pointer 69 * 70 * Calculate the size of the page directory in bytes (cayman+). 71 */ 72 static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev) 73 { 74 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8); 75 } 76 77 /** 78 * amdgpu_vm_get_bos - add the vm BOs to a validation list 79 * 80 * @vm: vm providing the BOs 81 * @head: head of validation list 82 * 83 * Add the page directory to the list of BOs to 84 * validate for command submission (cayman+). 85 */ 86 struct amdgpu_bo_list_entry *amdgpu_vm_get_bos(struct amdgpu_device *adev, 87 struct amdgpu_vm *vm, 88 struct list_head *head) 89 { 90 struct amdgpu_bo_list_entry *list; 91 unsigned i, idx; 92 93 list = drm_malloc_ab(vm->max_pde_used + 2, 94 sizeof(struct amdgpu_bo_list_entry)); 95 if (!list) { 96 return NULL; 97 } 98 99 /* add the vm page table to the list */ 100 list[0].robj = vm->page_directory; 101 list[0].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM; 102 list[0].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM; 103 list[0].priority = 0; 104 list[0].tv.bo = &vm->page_directory->tbo; 105 list[0].tv.shared = true; 106 list_add(&list[0].tv.head, head); 107 108 for (i = 0, idx = 1; i <= vm->max_pde_used; i++) { 109 if (!vm->page_tables[i].bo) 110 continue; 111 112 list[idx].robj = vm->page_tables[i].bo; 113 list[idx].prefered_domains = AMDGPU_GEM_DOMAIN_VRAM; 114 list[idx].allowed_domains = AMDGPU_GEM_DOMAIN_VRAM; 115 list[idx].priority = 0; 116 list[idx].tv.bo = &list[idx].robj->tbo; 117 list[idx].tv.shared = true; 118 list_add(&list[idx++].tv.head, head); 119 } 120 121 return list; 122 } 123 124 /** 125 * amdgpu_vm_grab_id - allocate the next free VMID 126 * 127 * @vm: vm to allocate id for 128 * @ring: ring we want to submit job to 129 * @sync: sync object where we add dependencies 130 * 131 * Allocate an id for the vm, adding fences to the sync obj as necessary. 132 * 133 * Global mutex must be locked! 134 */ 135 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, 136 struct amdgpu_sync *sync) 137 { 138 struct fence *best[AMDGPU_MAX_RINGS] = {}; 139 struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; 140 struct amdgpu_device *adev = ring->adev; 141 142 unsigned choices[2] = {}; 143 unsigned i; 144 145 /* check if the id is still valid */ 146 if (vm_id->id) { 147 unsigned id = vm_id->id; 148 long owner; 149 150 owner = atomic_long_read(&adev->vm_manager.ids[id].owner); 151 if (owner == (long)vm) { 152 trace_amdgpu_vm_grab_id(vm_id->id, ring->idx); 153 return 0; 154 } 155 } 156 157 /* we definately need to flush */ 158 vm_id->pd_gpu_addr = ~0ll; 159 160 /* skip over VMID 0, since it is the system VM */ 161 for (i = 1; i < adev->vm_manager.nvm; ++i) { 162 struct fence *fence = adev->vm_manager.ids[i].active; 163 struct amdgpu_ring *fring; 164 165 if (fence == NULL) { 166 /* found a free one */ 167 vm_id->id = i; 168 trace_amdgpu_vm_grab_id(i, ring->idx); 169 return 0; 170 } 171 172 fring = amdgpu_ring_from_fence(fence); 173 if (best[fring->idx] == NULL || 174 fence_is_later(best[fring->idx], fence)) { 175 best[fring->idx] = fence; 176 choices[fring == ring ? 0 : 1] = i; 177 } 178 } 179 180 for (i = 0; i < 2; ++i) { 181 if (choices[i]) { 182 struct fence *fence; 183 184 fence = adev->vm_manager.ids[choices[i]].active; 185 vm_id->id = choices[i]; 186 187 trace_amdgpu_vm_grab_id(choices[i], ring->idx); 188 return amdgpu_sync_fence(ring->adev, sync, fence); 189 } 190 } 191 192 /* should never happen */ 193 BUG(); 194 return -EINVAL; 195 } 196 197 /** 198 * amdgpu_vm_flush - hardware flush the vm 199 * 200 * @ring: ring to use for flush 201 * @vm: vm we want to flush 202 * @updates: last vm update that we waited for 203 * 204 * Flush the vm (cayman+). 205 * 206 * Global and local mutex must be locked! 207 */ 208 void amdgpu_vm_flush(struct amdgpu_ring *ring, 209 struct amdgpu_vm *vm, 210 struct fence *updates) 211 { 212 uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory); 213 struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; 214 struct fence *flushed_updates = vm_id->flushed_updates; 215 bool is_later; 216 217 if (!flushed_updates) 218 is_later = true; 219 else if (!updates) 220 is_later = false; 221 else 222 is_later = fence_is_later(updates, flushed_updates); 223 224 if (pd_addr != vm_id->pd_gpu_addr || is_later) { 225 trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id->id); 226 if (is_later) { 227 vm_id->flushed_updates = fence_get(updates); 228 fence_put(flushed_updates); 229 } 230 vm_id->pd_gpu_addr = pd_addr; 231 amdgpu_ring_emit_vm_flush(ring, vm_id->id, vm_id->pd_gpu_addr); 232 } 233 } 234 235 /** 236 * amdgpu_vm_fence - remember fence for vm 237 * 238 * @adev: amdgpu_device pointer 239 * @vm: vm we want to fence 240 * @fence: fence to remember 241 * 242 * Fence the vm (cayman+). 243 * Set the fence used to protect page table and id. 244 * 245 * Global and local mutex must be locked! 246 */ 247 void amdgpu_vm_fence(struct amdgpu_device *adev, 248 struct amdgpu_vm *vm, 249 struct fence *fence) 250 { 251 struct amdgpu_ring *ring = amdgpu_ring_from_fence(fence); 252 unsigned vm_id = vm->ids[ring->idx].id; 253 254 fence_put(adev->vm_manager.ids[vm_id].active); 255 adev->vm_manager.ids[vm_id].active = fence_get(fence); 256 atomic_long_set(&adev->vm_manager.ids[vm_id].owner, (long)vm); 257 } 258 259 /** 260 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo 261 * 262 * @vm: requested vm 263 * @bo: requested buffer object 264 * 265 * Find @bo inside the requested vm (cayman+). 266 * Search inside the @bos vm list for the requested vm 267 * Returns the found bo_va or NULL if none is found 268 * 269 * Object has to be reserved! 270 */ 271 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, 272 struct amdgpu_bo *bo) 273 { 274 struct amdgpu_bo_va *bo_va; 275 276 list_for_each_entry(bo_va, &bo->va, bo_list) { 277 if (bo_va->vm == vm) { 278 return bo_va; 279 } 280 } 281 return NULL; 282 } 283 284 /** 285 * amdgpu_vm_update_pages - helper to call the right asic function 286 * 287 * @adev: amdgpu_device pointer 288 * @ib: indirect buffer to fill with commands 289 * @pe: addr of the page entry 290 * @addr: dst addr to write into pe 291 * @count: number of page entries to update 292 * @incr: increase next addr by incr bytes 293 * @flags: hw access flags 294 * @gtt_flags: GTT hw access flags 295 * 296 * Traces the parameters and calls the right asic functions 297 * to setup the page table using the DMA. 298 */ 299 static void amdgpu_vm_update_pages(struct amdgpu_device *adev, 300 struct amdgpu_ib *ib, 301 uint64_t pe, uint64_t addr, 302 unsigned count, uint32_t incr, 303 uint32_t flags, uint32_t gtt_flags) 304 { 305 trace_amdgpu_vm_set_page(pe, addr, count, incr, flags); 306 307 if ((flags & AMDGPU_PTE_SYSTEM) && (flags == gtt_flags)) { 308 uint64_t src = adev->gart.table_addr + (addr >> 12) * 8; 309 amdgpu_vm_copy_pte(adev, ib, pe, src, count); 310 311 } else if ((flags & AMDGPU_PTE_SYSTEM) || (count < 3)) { 312 amdgpu_vm_write_pte(adev, ib, pe, addr, 313 count, incr, flags); 314 315 } else { 316 amdgpu_vm_set_pte_pde(adev, ib, pe, addr, 317 count, incr, flags); 318 } 319 } 320 321 int amdgpu_vm_free_job(struct amdgpu_job *job) 322 { 323 int i; 324 for (i = 0; i < job->num_ibs; i++) 325 amdgpu_ib_free(job->adev, &job->ibs[i]); 326 kfree(job->ibs); 327 return 0; 328 } 329 330 /** 331 * amdgpu_vm_clear_bo - initially clear the page dir/table 332 * 333 * @adev: amdgpu_device pointer 334 * @bo: bo to clear 335 * 336 * need to reserve bo first before calling it. 337 */ 338 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, 339 struct amdgpu_bo *bo) 340 { 341 struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; 342 struct fence *fence = NULL; 343 struct amdgpu_ib *ib; 344 unsigned entries; 345 uint64_t addr; 346 int r; 347 348 r = reservation_object_reserve_shared(bo->tbo.resv); 349 if (r) 350 return r; 351 352 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); 353 if (r) 354 goto error; 355 356 addr = amdgpu_bo_gpu_offset(bo); 357 entries = amdgpu_bo_size(bo) / 8; 358 359 ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); 360 if (!ib) 361 goto error; 362 363 r = amdgpu_ib_get(ring, NULL, entries * 2 + 64, ib); 364 if (r) 365 goto error_free; 366 367 ib->length_dw = 0; 368 369 amdgpu_vm_update_pages(adev, ib, addr, 0, entries, 0, 0, 0); 370 amdgpu_vm_pad_ib(adev, ib); 371 WARN_ON(ib->length_dw > 64); 372 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, 373 &amdgpu_vm_free_job, 374 AMDGPU_FENCE_OWNER_VM, 375 &fence); 376 if (!r) 377 amdgpu_bo_fence(bo, fence, true); 378 fence_put(fence); 379 if (amdgpu_enable_scheduler) 380 return 0; 381 382 error_free: 383 amdgpu_ib_free(adev, ib); 384 kfree(ib); 385 386 error: 387 return r; 388 } 389 390 /** 391 * amdgpu_vm_map_gart - get the physical address of a gart page 392 * 393 * @adev: amdgpu_device pointer 394 * @addr: the unmapped addr 395 * 396 * Look up the physical address of the page that the pte resolves 397 * to (cayman+). 398 * Returns the physical address of the page. 399 */ 400 uint64_t amdgpu_vm_map_gart(struct amdgpu_device *adev, uint64_t addr) 401 { 402 uint64_t result; 403 404 /* page table offset */ 405 result = adev->gart.pages_addr[addr >> PAGE_SHIFT]; 406 407 /* in case cpu page size != gpu page size*/ 408 result |= addr & (~PAGE_MASK); 409 410 return result; 411 } 412 413 /** 414 * amdgpu_vm_update_pdes - make sure that page directory is valid 415 * 416 * @adev: amdgpu_device pointer 417 * @vm: requested vm 418 * @start: start of GPU address range 419 * @end: end of GPU address range 420 * 421 * Allocates new page tables if necessary 422 * and updates the page directory (cayman+). 423 * Returns 0 for success, error for failure. 424 * 425 * Global and local mutex must be locked! 426 */ 427 int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, 428 struct amdgpu_vm *vm) 429 { 430 struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; 431 struct amdgpu_bo *pd = vm->page_directory; 432 uint64_t pd_addr = amdgpu_bo_gpu_offset(pd); 433 uint32_t incr = AMDGPU_VM_PTE_COUNT * 8; 434 uint64_t last_pde = ~0, last_pt = ~0; 435 unsigned count = 0, pt_idx, ndw; 436 struct amdgpu_ib *ib; 437 struct fence *fence = NULL; 438 439 int r; 440 441 /* padding, etc. */ 442 ndw = 64; 443 444 /* assume the worst case */ 445 ndw += vm->max_pde_used * 6; 446 447 /* update too big for an IB */ 448 if (ndw > 0xfffff) 449 return -ENOMEM; 450 451 ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); 452 if (!ib) 453 return -ENOMEM; 454 455 r = amdgpu_ib_get(ring, NULL, ndw * 4, ib); 456 if (r) { 457 kfree(ib); 458 return r; 459 } 460 ib->length_dw = 0; 461 462 /* walk over the address space and update the page directory */ 463 for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) { 464 struct amdgpu_bo *bo = vm->page_tables[pt_idx].bo; 465 uint64_t pde, pt; 466 467 if (bo == NULL) 468 continue; 469 470 pt = amdgpu_bo_gpu_offset(bo); 471 if (vm->page_tables[pt_idx].addr == pt) 472 continue; 473 vm->page_tables[pt_idx].addr = pt; 474 475 pde = pd_addr + pt_idx * 8; 476 if (((last_pde + 8 * count) != pde) || 477 ((last_pt + incr * count) != pt)) { 478 479 if (count) { 480 amdgpu_vm_update_pages(adev, ib, last_pde, 481 last_pt, count, incr, 482 AMDGPU_PTE_VALID, 0); 483 } 484 485 count = 1; 486 last_pde = pde; 487 last_pt = pt; 488 } else { 489 ++count; 490 } 491 } 492 493 if (count) 494 amdgpu_vm_update_pages(adev, ib, last_pde, last_pt, count, 495 incr, AMDGPU_PTE_VALID, 0); 496 497 if (ib->length_dw != 0) { 498 amdgpu_vm_pad_ib(adev, ib); 499 amdgpu_sync_resv(adev, &ib->sync, pd->tbo.resv, AMDGPU_FENCE_OWNER_VM); 500 WARN_ON(ib->length_dw > ndw); 501 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, 502 &amdgpu_vm_free_job, 503 AMDGPU_FENCE_OWNER_VM, 504 &fence); 505 if (r) 506 goto error_free; 507 508 amdgpu_bo_fence(pd, fence, true); 509 fence_put(vm->page_directory_fence); 510 vm->page_directory_fence = fence_get(fence); 511 fence_put(fence); 512 } 513 514 if (!amdgpu_enable_scheduler || ib->length_dw == 0) { 515 amdgpu_ib_free(adev, ib); 516 kfree(ib); 517 } 518 519 return 0; 520 521 error_free: 522 amdgpu_ib_free(adev, ib); 523 kfree(ib); 524 return r; 525 } 526 527 /** 528 * amdgpu_vm_frag_ptes - add fragment information to PTEs 529 * 530 * @adev: amdgpu_device pointer 531 * @ib: IB for the update 532 * @pe_start: first PTE to handle 533 * @pe_end: last PTE to handle 534 * @addr: addr those PTEs should point to 535 * @flags: hw mapping flags 536 * @gtt_flags: GTT hw mapping flags 537 * 538 * Global and local mutex must be locked! 539 */ 540 static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev, 541 struct amdgpu_ib *ib, 542 uint64_t pe_start, uint64_t pe_end, 543 uint64_t addr, uint32_t flags, 544 uint32_t gtt_flags) 545 { 546 /** 547 * The MC L1 TLB supports variable sized pages, based on a fragment 548 * field in the PTE. When this field is set to a non-zero value, page 549 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE 550 * flags are considered valid for all PTEs within the fragment range 551 * and corresponding mappings are assumed to be physically contiguous. 552 * 553 * The L1 TLB can store a single PTE for the whole fragment, 554 * significantly increasing the space available for translation 555 * caching. This leads to large improvements in throughput when the 556 * TLB is under pressure. 557 * 558 * The L2 TLB distributes small and large fragments into two 559 * asymmetric partitions. The large fragment cache is significantly 560 * larger. Thus, we try to use large fragments wherever possible. 561 * Userspace can support this by aligning virtual base address and 562 * allocation size to the fragment size. 563 */ 564 565 /* SI and newer are optimized for 64KB */ 566 uint64_t frag_flags = AMDGPU_PTE_FRAG_64KB; 567 uint64_t frag_align = 0x80; 568 569 uint64_t frag_start = ALIGN(pe_start, frag_align); 570 uint64_t frag_end = pe_end & ~(frag_align - 1); 571 572 unsigned count; 573 574 /* system pages are non continuously */ 575 if ((flags & AMDGPU_PTE_SYSTEM) || !(flags & AMDGPU_PTE_VALID) || 576 (frag_start >= frag_end)) { 577 578 count = (pe_end - pe_start) / 8; 579 amdgpu_vm_update_pages(adev, ib, pe_start, addr, count, 580 AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); 581 return; 582 } 583 584 /* handle the 4K area at the beginning */ 585 if (pe_start != frag_start) { 586 count = (frag_start - pe_start) / 8; 587 amdgpu_vm_update_pages(adev, ib, pe_start, addr, count, 588 AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); 589 addr += AMDGPU_GPU_PAGE_SIZE * count; 590 } 591 592 /* handle the area in the middle */ 593 count = (frag_end - frag_start) / 8; 594 amdgpu_vm_update_pages(adev, ib, frag_start, addr, count, 595 AMDGPU_GPU_PAGE_SIZE, flags | frag_flags, 596 gtt_flags); 597 598 /* handle the 4K area at the end */ 599 if (frag_end != pe_end) { 600 addr += AMDGPU_GPU_PAGE_SIZE * count; 601 count = (pe_end - frag_end) / 8; 602 amdgpu_vm_update_pages(adev, ib, frag_end, addr, count, 603 AMDGPU_GPU_PAGE_SIZE, flags, gtt_flags); 604 } 605 } 606 607 /** 608 * amdgpu_vm_update_ptes - make sure that page tables are valid 609 * 610 * @adev: amdgpu_device pointer 611 * @vm: requested vm 612 * @start: start of GPU address range 613 * @end: end of GPU address range 614 * @dst: destination address to map to 615 * @flags: mapping flags 616 * 617 * Update the page tables in the range @start - @end (cayman+). 618 * 619 * Global and local mutex must be locked! 620 */ 621 static int amdgpu_vm_update_ptes(struct amdgpu_device *adev, 622 struct amdgpu_vm *vm, 623 struct amdgpu_ib *ib, 624 uint64_t start, uint64_t end, 625 uint64_t dst, uint32_t flags, 626 uint32_t gtt_flags) 627 { 628 uint64_t mask = AMDGPU_VM_PTE_COUNT - 1; 629 uint64_t last_pte = ~0, last_dst = ~0; 630 void *owner = AMDGPU_FENCE_OWNER_VM; 631 unsigned count = 0; 632 uint64_t addr; 633 634 /* sync to everything on unmapping */ 635 if (!(flags & AMDGPU_PTE_VALID)) 636 owner = AMDGPU_FENCE_OWNER_UNDEFINED; 637 638 /* walk over the address space and update the page tables */ 639 for (addr = start; addr < end; ) { 640 uint64_t pt_idx = addr >> amdgpu_vm_block_size; 641 struct amdgpu_bo *pt = vm->page_tables[pt_idx].bo; 642 unsigned nptes; 643 uint64_t pte; 644 int r; 645 646 amdgpu_sync_resv(adev, &ib->sync, pt->tbo.resv, owner); 647 r = reservation_object_reserve_shared(pt->tbo.resv); 648 if (r) 649 return r; 650 651 if ((addr & ~mask) == (end & ~mask)) 652 nptes = end - addr; 653 else 654 nptes = AMDGPU_VM_PTE_COUNT - (addr & mask); 655 656 pte = amdgpu_bo_gpu_offset(pt); 657 pte += (addr & mask) * 8; 658 659 if ((last_pte + 8 * count) != pte) { 660 661 if (count) { 662 amdgpu_vm_frag_ptes(adev, ib, last_pte, 663 last_pte + 8 * count, 664 last_dst, flags, 665 gtt_flags); 666 } 667 668 count = nptes; 669 last_pte = pte; 670 last_dst = dst; 671 } else { 672 count += nptes; 673 } 674 675 addr += nptes; 676 dst += nptes * AMDGPU_GPU_PAGE_SIZE; 677 } 678 679 if (count) { 680 amdgpu_vm_frag_ptes(adev, ib, last_pte, 681 last_pte + 8 * count, 682 last_dst, flags, gtt_flags); 683 } 684 685 return 0; 686 } 687 688 /** 689 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table 690 * 691 * @adev: amdgpu_device pointer 692 * @vm: requested vm 693 * @mapping: mapped range and flags to use for the update 694 * @addr: addr to set the area to 695 * @gtt_flags: flags as they are used for GTT 696 * @fence: optional resulting fence 697 * 698 * Fill in the page table entries for @mapping. 699 * Returns 0 for success, -EINVAL for failure. 700 * 701 * Object have to be reserved and mutex must be locked! 702 */ 703 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, 704 struct amdgpu_vm *vm, 705 struct amdgpu_bo_va_mapping *mapping, 706 uint64_t addr, uint32_t gtt_flags, 707 struct fence **fence) 708 { 709 struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; 710 unsigned nptes, ncmds, ndw; 711 uint32_t flags = gtt_flags; 712 struct amdgpu_ib *ib; 713 struct fence *f = NULL; 714 int r; 715 716 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here 717 * but in case of something, we filter the flags in first place 718 */ 719 if (!(mapping->flags & AMDGPU_PTE_READABLE)) 720 flags &= ~AMDGPU_PTE_READABLE; 721 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) 722 flags &= ~AMDGPU_PTE_WRITEABLE; 723 724 trace_amdgpu_vm_bo_update(mapping); 725 726 nptes = mapping->it.last - mapping->it.start + 1; 727 728 /* 729 * reserve space for one command every (1 << BLOCK_SIZE) 730 * entries or 2k dwords (whatever is smaller) 731 */ 732 ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1; 733 734 /* padding, etc. */ 735 ndw = 64; 736 737 if ((flags & AMDGPU_PTE_SYSTEM) && (flags == gtt_flags)) { 738 /* only copy commands needed */ 739 ndw += ncmds * 7; 740 741 } else if (flags & AMDGPU_PTE_SYSTEM) { 742 /* header for write data commands */ 743 ndw += ncmds * 4; 744 745 /* body of write data command */ 746 ndw += nptes * 2; 747 748 } else { 749 /* set page commands needed */ 750 ndw += ncmds * 10; 751 752 /* two extra commands for begin/end of fragment */ 753 ndw += 2 * 10; 754 } 755 756 /* update too big for an IB */ 757 if (ndw > 0xfffff) 758 return -ENOMEM; 759 760 ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); 761 if (!ib) 762 return -ENOMEM; 763 764 r = amdgpu_ib_get(ring, NULL, ndw * 4, ib); 765 if (r) { 766 kfree(ib); 767 return r; 768 } 769 770 ib->length_dw = 0; 771 772 r = amdgpu_vm_update_ptes(adev, vm, ib, mapping->it.start, 773 mapping->it.last + 1, addr + mapping->offset, 774 flags, gtt_flags); 775 776 if (r) { 777 amdgpu_ib_free(adev, ib); 778 kfree(ib); 779 return r; 780 } 781 782 amdgpu_vm_pad_ib(adev, ib); 783 WARN_ON(ib->length_dw > ndw); 784 r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, 785 &amdgpu_vm_free_job, 786 AMDGPU_FENCE_OWNER_VM, 787 &f); 788 if (r) 789 goto error_free; 790 791 amdgpu_bo_fence(vm->page_directory, f, true); 792 if (fence) { 793 fence_put(*fence); 794 *fence = fence_get(f); 795 } 796 fence_put(f); 797 if (!amdgpu_enable_scheduler) { 798 amdgpu_ib_free(adev, ib); 799 kfree(ib); 800 } 801 return 0; 802 803 error_free: 804 amdgpu_ib_free(adev, ib); 805 kfree(ib); 806 return r; 807 } 808 809 /** 810 * amdgpu_vm_bo_update - update all BO mappings in the vm page table 811 * 812 * @adev: amdgpu_device pointer 813 * @bo_va: requested BO and VM object 814 * @mem: ttm mem 815 * 816 * Fill in the page table entries for @bo_va. 817 * Returns 0 for success, -EINVAL for failure. 818 * 819 * Object have to be reserved and mutex must be locked! 820 */ 821 int amdgpu_vm_bo_update(struct amdgpu_device *adev, 822 struct amdgpu_bo_va *bo_va, 823 struct ttm_mem_reg *mem) 824 { 825 struct amdgpu_vm *vm = bo_va->vm; 826 struct amdgpu_bo_va_mapping *mapping; 827 uint32_t flags; 828 uint64_t addr; 829 int r; 830 831 if (mem) { 832 addr = (u64)mem->start << PAGE_SHIFT; 833 if (mem->mem_type != TTM_PL_TT) 834 addr += adev->vm_manager.vram_base_offset; 835 } else { 836 addr = 0; 837 } 838 839 flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem); 840 841 spin_lock(&vm->status_lock); 842 if (!list_empty(&bo_va->vm_status)) 843 list_splice_init(&bo_va->valids, &bo_va->invalids); 844 spin_unlock(&vm->status_lock); 845 846 list_for_each_entry(mapping, &bo_va->invalids, list) { 847 r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, addr, 848 flags, &bo_va->last_pt_update); 849 if (r) 850 return r; 851 } 852 853 if (trace_amdgpu_vm_bo_mapping_enabled()) { 854 list_for_each_entry(mapping, &bo_va->valids, list) 855 trace_amdgpu_vm_bo_mapping(mapping); 856 857 list_for_each_entry(mapping, &bo_va->invalids, list) 858 trace_amdgpu_vm_bo_mapping(mapping); 859 } 860 861 spin_lock(&vm->status_lock); 862 list_splice_init(&bo_va->invalids, &bo_va->valids); 863 list_del_init(&bo_va->vm_status); 864 if (!mem) 865 list_add(&bo_va->vm_status, &vm->cleared); 866 spin_unlock(&vm->status_lock); 867 868 return 0; 869 } 870 871 /** 872 * amdgpu_vm_clear_freed - clear freed BOs in the PT 873 * 874 * @adev: amdgpu_device pointer 875 * @vm: requested vm 876 * 877 * Make sure all freed BOs are cleared in the PT. 878 * Returns 0 for success. 879 * 880 * PTs have to be reserved and mutex must be locked! 881 */ 882 int amdgpu_vm_clear_freed(struct amdgpu_device *adev, 883 struct amdgpu_vm *vm) 884 { 885 struct amdgpu_bo_va_mapping *mapping; 886 int r; 887 888 while (!list_empty(&vm->freed)) { 889 mapping = list_first_entry(&vm->freed, 890 struct amdgpu_bo_va_mapping, list); 891 list_del(&mapping->list); 892 893 r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, 0, 0, NULL); 894 kfree(mapping); 895 if (r) 896 return r; 897 898 } 899 return 0; 900 901 } 902 903 /** 904 * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT 905 * 906 * @adev: amdgpu_device pointer 907 * @vm: requested vm 908 * 909 * Make sure all invalidated BOs are cleared in the PT. 910 * Returns 0 for success. 911 * 912 * PTs have to be reserved and mutex must be locked! 913 */ 914 int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, 915 struct amdgpu_vm *vm, struct amdgpu_sync *sync) 916 { 917 struct amdgpu_bo_va *bo_va = NULL; 918 int r = 0; 919 920 spin_lock(&vm->status_lock); 921 while (!list_empty(&vm->invalidated)) { 922 bo_va = list_first_entry(&vm->invalidated, 923 struct amdgpu_bo_va, vm_status); 924 spin_unlock(&vm->status_lock); 925 926 r = amdgpu_vm_bo_update(adev, bo_va, NULL); 927 if (r) 928 return r; 929 930 spin_lock(&vm->status_lock); 931 } 932 spin_unlock(&vm->status_lock); 933 934 if (bo_va) 935 r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update); 936 937 return r; 938 } 939 940 /** 941 * amdgpu_vm_bo_add - add a bo to a specific vm 942 * 943 * @adev: amdgpu_device pointer 944 * @vm: requested vm 945 * @bo: amdgpu buffer object 946 * 947 * Add @bo into the requested vm (cayman+). 948 * Add @bo to the list of bos associated with the vm 949 * Returns newly added bo_va or NULL for failure 950 * 951 * Object has to be reserved! 952 */ 953 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, 954 struct amdgpu_vm *vm, 955 struct amdgpu_bo *bo) 956 { 957 struct amdgpu_bo_va *bo_va; 958 959 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL); 960 if (bo_va == NULL) { 961 return NULL; 962 } 963 bo_va->vm = vm; 964 bo_va->bo = bo; 965 bo_va->ref_count = 1; 966 INIT_LIST_HEAD(&bo_va->bo_list); 967 INIT_LIST_HEAD(&bo_va->valids); 968 INIT_LIST_HEAD(&bo_va->invalids); 969 INIT_LIST_HEAD(&bo_va->vm_status); 970 971 list_add_tail(&bo_va->bo_list, &bo->va); 972 973 return bo_va; 974 } 975 976 /** 977 * amdgpu_vm_bo_map - map bo inside a vm 978 * 979 * @adev: amdgpu_device pointer 980 * @bo_va: bo_va to store the address 981 * @saddr: where to map the BO 982 * @offset: requested offset in the BO 983 * @flags: attributes of pages (read/write/valid/etc.) 984 * 985 * Add a mapping of the BO at the specefied addr into the VM. 986 * Returns 0 for success, error for failure. 987 * 988 * Object has to be reserved and unreserved outside! 989 */ 990 int amdgpu_vm_bo_map(struct amdgpu_device *adev, 991 struct amdgpu_bo_va *bo_va, 992 uint64_t saddr, uint64_t offset, 993 uint64_t size, uint32_t flags) 994 { 995 struct amdgpu_bo_va_mapping *mapping; 996 struct amdgpu_vm *vm = bo_va->vm; 997 struct interval_tree_node *it; 998 unsigned last_pfn, pt_idx; 999 uint64_t eaddr; 1000 int r; 1001 1002 /* validate the parameters */ 1003 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || 1004 size == 0 || size & AMDGPU_GPU_PAGE_MASK) 1005 return -EINVAL; 1006 1007 /* make sure object fit at this offset */ 1008 eaddr = saddr + size; 1009 if ((saddr >= eaddr) || (offset + size > amdgpu_bo_size(bo_va->bo))) 1010 return -EINVAL; 1011 1012 last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE; 1013 if (last_pfn > adev->vm_manager.max_pfn) { 1014 dev_err(adev->dev, "va above limit (0x%08X > 0x%08X)\n", 1015 last_pfn, adev->vm_manager.max_pfn); 1016 return -EINVAL; 1017 } 1018 1019 saddr /= AMDGPU_GPU_PAGE_SIZE; 1020 eaddr /= AMDGPU_GPU_PAGE_SIZE; 1021 1022 spin_lock(&vm->it_lock); 1023 it = interval_tree_iter_first(&vm->va, saddr, eaddr - 1); 1024 spin_unlock(&vm->it_lock); 1025 if (it) { 1026 struct amdgpu_bo_va_mapping *tmp; 1027 tmp = container_of(it, struct amdgpu_bo_va_mapping, it); 1028 /* bo and tmp overlap, invalid addr */ 1029 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with " 1030 "0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr, 1031 tmp->it.start, tmp->it.last + 1); 1032 r = -EINVAL; 1033 goto error; 1034 } 1035 1036 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 1037 if (!mapping) { 1038 r = -ENOMEM; 1039 goto error; 1040 } 1041 1042 INIT_LIST_HEAD(&mapping->list); 1043 mapping->it.start = saddr; 1044 mapping->it.last = eaddr - 1; 1045 mapping->offset = offset; 1046 mapping->flags = flags; 1047 1048 list_add(&mapping->list, &bo_va->invalids); 1049 spin_lock(&vm->it_lock); 1050 interval_tree_insert(&mapping->it, &vm->va); 1051 spin_unlock(&vm->it_lock); 1052 trace_amdgpu_vm_bo_map(bo_va, mapping); 1053 1054 /* Make sure the page tables are allocated */ 1055 saddr >>= amdgpu_vm_block_size; 1056 eaddr >>= amdgpu_vm_block_size; 1057 1058 BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev)); 1059 1060 if (eaddr > vm->max_pde_used) 1061 vm->max_pde_used = eaddr; 1062 1063 /* walk over the address space and allocate the page tables */ 1064 for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) { 1065 struct reservation_object *resv = vm->page_directory->tbo.resv; 1066 struct amdgpu_bo *pt; 1067 1068 if (vm->page_tables[pt_idx].bo) 1069 continue; 1070 1071 r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8, 1072 AMDGPU_GPU_PAGE_SIZE, true, 1073 AMDGPU_GEM_DOMAIN_VRAM, 1074 AMDGPU_GEM_CREATE_NO_CPU_ACCESS, 1075 NULL, resv, &pt); 1076 if (r) 1077 goto error_free; 1078 1079 r = amdgpu_vm_clear_bo(adev, pt); 1080 if (r) { 1081 amdgpu_bo_unref(&pt); 1082 goto error_free; 1083 } 1084 1085 vm->page_tables[pt_idx].addr = 0; 1086 vm->page_tables[pt_idx].bo = pt; 1087 } 1088 1089 return 0; 1090 1091 error_free: 1092 list_del(&mapping->list); 1093 spin_lock(&vm->it_lock); 1094 interval_tree_remove(&mapping->it, &vm->va); 1095 spin_unlock(&vm->it_lock); 1096 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1097 kfree(mapping); 1098 1099 error: 1100 return r; 1101 } 1102 1103 /** 1104 * amdgpu_vm_bo_unmap - remove bo mapping from vm 1105 * 1106 * @adev: amdgpu_device pointer 1107 * @bo_va: bo_va to remove the address from 1108 * @saddr: where to the BO is mapped 1109 * 1110 * Remove a mapping of the BO at the specefied addr from the VM. 1111 * Returns 0 for success, error for failure. 1112 * 1113 * Object has to be reserved and unreserved outside! 1114 */ 1115 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, 1116 struct amdgpu_bo_va *bo_va, 1117 uint64_t saddr) 1118 { 1119 struct amdgpu_bo_va_mapping *mapping; 1120 struct amdgpu_vm *vm = bo_va->vm; 1121 bool valid = true; 1122 1123 saddr /= AMDGPU_GPU_PAGE_SIZE; 1124 1125 list_for_each_entry(mapping, &bo_va->valids, list) { 1126 if (mapping->it.start == saddr) 1127 break; 1128 } 1129 1130 if (&mapping->list == &bo_va->valids) { 1131 valid = false; 1132 1133 list_for_each_entry(mapping, &bo_va->invalids, list) { 1134 if (mapping->it.start == saddr) 1135 break; 1136 } 1137 1138 if (&mapping->list == &bo_va->invalids) 1139 return -ENOENT; 1140 } 1141 1142 list_del(&mapping->list); 1143 spin_lock(&vm->it_lock); 1144 interval_tree_remove(&mapping->it, &vm->va); 1145 spin_unlock(&vm->it_lock); 1146 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1147 1148 if (valid) 1149 list_add(&mapping->list, &vm->freed); 1150 else 1151 kfree(mapping); 1152 1153 return 0; 1154 } 1155 1156 /** 1157 * amdgpu_vm_bo_rmv - remove a bo to a specific vm 1158 * 1159 * @adev: amdgpu_device pointer 1160 * @bo_va: requested bo_va 1161 * 1162 * Remove @bo_va->bo from the requested vm (cayman+). 1163 * 1164 * Object have to be reserved! 1165 */ 1166 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, 1167 struct amdgpu_bo_va *bo_va) 1168 { 1169 struct amdgpu_bo_va_mapping *mapping, *next; 1170 struct amdgpu_vm *vm = bo_va->vm; 1171 1172 list_del(&bo_va->bo_list); 1173 1174 spin_lock(&vm->status_lock); 1175 list_del(&bo_va->vm_status); 1176 spin_unlock(&vm->status_lock); 1177 1178 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 1179 list_del(&mapping->list); 1180 spin_lock(&vm->it_lock); 1181 interval_tree_remove(&mapping->it, &vm->va); 1182 spin_unlock(&vm->it_lock); 1183 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 1184 list_add(&mapping->list, &vm->freed); 1185 } 1186 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { 1187 list_del(&mapping->list); 1188 spin_lock(&vm->it_lock); 1189 interval_tree_remove(&mapping->it, &vm->va); 1190 spin_unlock(&vm->it_lock); 1191 kfree(mapping); 1192 } 1193 1194 fence_put(bo_va->last_pt_update); 1195 kfree(bo_va); 1196 } 1197 1198 /** 1199 * amdgpu_vm_bo_invalidate - mark the bo as invalid 1200 * 1201 * @adev: amdgpu_device pointer 1202 * @vm: requested vm 1203 * @bo: amdgpu buffer object 1204 * 1205 * Mark @bo as invalid (cayman+). 1206 */ 1207 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, 1208 struct amdgpu_bo *bo) 1209 { 1210 struct amdgpu_bo_va *bo_va; 1211 1212 list_for_each_entry(bo_va, &bo->va, bo_list) { 1213 spin_lock(&bo_va->vm->status_lock); 1214 if (list_empty(&bo_va->vm_status)) 1215 list_add(&bo_va->vm_status, &bo_va->vm->invalidated); 1216 spin_unlock(&bo_va->vm->status_lock); 1217 } 1218 } 1219 1220 /** 1221 * amdgpu_vm_init - initialize a vm instance 1222 * 1223 * @adev: amdgpu_device pointer 1224 * @vm: requested vm 1225 * 1226 * Init @vm fields (cayman+). 1227 */ 1228 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1229 { 1230 const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE, 1231 AMDGPU_VM_PTE_COUNT * 8); 1232 unsigned pd_size, pd_entries, pts_size; 1233 int i, r; 1234 1235 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 1236 vm->ids[i].id = 0; 1237 vm->ids[i].flushed_updates = NULL; 1238 } 1239 mutex_init(&vm->mutex); 1240 vm->va = RB_ROOT; 1241 spin_lock_init(&vm->status_lock); 1242 INIT_LIST_HEAD(&vm->invalidated); 1243 INIT_LIST_HEAD(&vm->cleared); 1244 INIT_LIST_HEAD(&vm->freed); 1245 spin_lock_init(&vm->it_lock); 1246 pd_size = amdgpu_vm_directory_size(adev); 1247 pd_entries = amdgpu_vm_num_pdes(adev); 1248 1249 /* allocate page table array */ 1250 pts_size = pd_entries * sizeof(struct amdgpu_vm_pt); 1251 vm->page_tables = kzalloc(pts_size, GFP_KERNEL); 1252 if (vm->page_tables == NULL) { 1253 DRM_ERROR("Cannot allocate memory for page table array\n"); 1254 return -ENOMEM; 1255 } 1256 1257 vm->page_directory_fence = NULL; 1258 1259 r = amdgpu_bo_create(adev, pd_size, align, true, 1260 AMDGPU_GEM_DOMAIN_VRAM, 1261 AMDGPU_GEM_CREATE_NO_CPU_ACCESS, 1262 NULL, NULL, &vm->page_directory); 1263 if (r) 1264 return r; 1265 r = amdgpu_bo_reserve(vm->page_directory, false); 1266 if (r) { 1267 amdgpu_bo_unref(&vm->page_directory); 1268 vm->page_directory = NULL; 1269 return r; 1270 } 1271 r = amdgpu_vm_clear_bo(adev, vm->page_directory); 1272 amdgpu_bo_unreserve(vm->page_directory); 1273 if (r) { 1274 amdgpu_bo_unref(&vm->page_directory); 1275 vm->page_directory = NULL; 1276 return r; 1277 } 1278 1279 return 0; 1280 } 1281 1282 /** 1283 * amdgpu_vm_fini - tear down a vm instance 1284 * 1285 * @adev: amdgpu_device pointer 1286 * @vm: requested vm 1287 * 1288 * Tear down @vm (cayman+). 1289 * Unbind the VM and remove all bos from the vm bo list 1290 */ 1291 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1292 { 1293 struct amdgpu_bo_va_mapping *mapping, *tmp; 1294 int i; 1295 1296 if (!RB_EMPTY_ROOT(&vm->va)) { 1297 dev_err(adev->dev, "still active bo inside vm\n"); 1298 } 1299 rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) { 1300 list_del(&mapping->list); 1301 interval_tree_remove(&mapping->it, &vm->va); 1302 kfree(mapping); 1303 } 1304 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { 1305 list_del(&mapping->list); 1306 kfree(mapping); 1307 } 1308 1309 for (i = 0; i < amdgpu_vm_num_pdes(adev); i++) 1310 amdgpu_bo_unref(&vm->page_tables[i].bo); 1311 kfree(vm->page_tables); 1312 1313 amdgpu_bo_unref(&vm->page_directory); 1314 fence_put(vm->page_directory_fence); 1315 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 1316 unsigned id = vm->ids[i].id; 1317 1318 atomic_long_cmpxchg(&adev->vm_manager.ids[id].owner, 1319 (long)vm, 0); 1320 fence_put(vm->ids[i].flushed_updates); 1321 } 1322 1323 mutex_destroy(&vm->mutex); 1324 } 1325 1326 /** 1327 * amdgpu_vm_manager_fini - cleanup VM manager 1328 * 1329 * @adev: amdgpu_device pointer 1330 * 1331 * Cleanup the VM manager and free resources. 1332 */ 1333 void amdgpu_vm_manager_fini(struct amdgpu_device *adev) 1334 { 1335 unsigned i; 1336 1337 for (i = 0; i < AMDGPU_NUM_VM; ++i) 1338 fence_put(adev->vm_manager.ids[i].active); 1339 } 1340