1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <linux/dma-fence-array.h> 29 #include <linux/interval_tree_generic.h> 30 #include <linux/idr.h> 31 32 #include <drm/amdgpu_drm.h> 33 #include "amdgpu.h" 34 #include "amdgpu_trace.h" 35 #include "amdgpu_amdkfd.h" 36 #include "amdgpu_gmc.h" 37 #include "amdgpu_xgmi.h" 38 39 /** 40 * DOC: GPUVM 41 * 42 * GPUVM is similar to the legacy gart on older asics, however 43 * rather than there being a single global gart table 44 * for the entire GPU, there are multiple VM page tables active 45 * at any given time. The VM page tables can contain a mix 46 * vram pages and system memory pages and system memory pages 47 * can be mapped as snooped (cached system pages) or unsnooped 48 * (uncached system pages). 49 * Each VM has an ID associated with it and there is a page table 50 * associated with each VMID. When execting a command buffer, 51 * the kernel tells the the ring what VMID to use for that command 52 * buffer. VMIDs are allocated dynamically as commands are submitted. 53 * The userspace drivers maintain their own address space and the kernel 54 * sets up their pages tables accordingly when they submit their 55 * command buffers and a VMID is assigned. 56 * Cayman/Trinity support up to 8 active VMs at any given time; 57 * SI supports 16. 58 */ 59 60 #define START(node) ((node)->start) 61 #define LAST(node) ((node)->last) 62 63 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, 64 START, LAST, static, amdgpu_vm_it) 65 66 #undef START 67 #undef LAST 68 69 /** 70 * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback 71 */ 72 struct amdgpu_prt_cb { 73 74 /** 75 * @adev: amdgpu device 76 */ 77 struct amdgpu_device *adev; 78 79 /** 80 * @cb: callback 81 */ 82 struct dma_fence_cb cb; 83 }; 84 85 /** 86 * amdgpu_vm_level_shift - return the addr shift for each level 87 * 88 * @adev: amdgpu_device pointer 89 * @level: VMPT level 90 * 91 * Returns: 92 * The number of bits the pfn needs to be right shifted for a level. 93 */ 94 static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev, 95 unsigned level) 96 { 97 unsigned shift = 0xff; 98 99 switch (level) { 100 case AMDGPU_VM_PDB2: 101 case AMDGPU_VM_PDB1: 102 case AMDGPU_VM_PDB0: 103 shift = 9 * (AMDGPU_VM_PDB0 - level) + 104 adev->vm_manager.block_size; 105 break; 106 case AMDGPU_VM_PTB: 107 shift = 0; 108 break; 109 default: 110 dev_err(adev->dev, "the level%d isn't supported.\n", level); 111 } 112 113 return shift; 114 } 115 116 /** 117 * amdgpu_vm_num_entries - return the number of entries in a PD/PT 118 * 119 * @adev: amdgpu_device pointer 120 * @level: VMPT level 121 * 122 * Returns: 123 * The number of entries in a page directory or page table. 124 */ 125 static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev, 126 unsigned level) 127 { 128 unsigned shift = amdgpu_vm_level_shift(adev, 129 adev->vm_manager.root_level); 130 131 if (level == adev->vm_manager.root_level) 132 /* For the root directory */ 133 return round_up(adev->vm_manager.max_pfn, 1ULL << shift) >> shift; 134 else if (level != AMDGPU_VM_PTB) 135 /* Everything in between */ 136 return 512; 137 else 138 /* For the page tables on the leaves */ 139 return AMDGPU_VM_PTE_COUNT(adev); 140 } 141 142 /** 143 * amdgpu_vm_num_ats_entries - return the number of ATS entries in the root PD 144 * 145 * @adev: amdgpu_device pointer 146 * 147 * Returns: 148 * The number of entries in the root page directory which needs the ATS setting. 149 */ 150 static unsigned amdgpu_vm_num_ats_entries(struct amdgpu_device *adev) 151 { 152 unsigned shift; 153 154 shift = amdgpu_vm_level_shift(adev, adev->vm_manager.root_level); 155 return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT); 156 } 157 158 /** 159 * amdgpu_vm_entries_mask - the mask to get the entry number of a PD/PT 160 * 161 * @adev: amdgpu_device pointer 162 * @level: VMPT level 163 * 164 * Returns: 165 * The mask to extract the entry number of a PD/PT from an address. 166 */ 167 static uint32_t amdgpu_vm_entries_mask(struct amdgpu_device *adev, 168 unsigned int level) 169 { 170 if (level <= adev->vm_manager.root_level) 171 return 0xffffffff; 172 else if (level != AMDGPU_VM_PTB) 173 return 0x1ff; 174 else 175 return AMDGPU_VM_PTE_COUNT(adev) - 1; 176 } 177 178 /** 179 * amdgpu_vm_bo_size - returns the size of the BOs in bytes 180 * 181 * @adev: amdgpu_device pointer 182 * @level: VMPT level 183 * 184 * Returns: 185 * The size of the BO for a page directory or page table in bytes. 186 */ 187 static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level) 188 { 189 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8); 190 } 191 192 /** 193 * amdgpu_vm_bo_evicted - vm_bo is evicted 194 * 195 * @vm_bo: vm_bo which is evicted 196 * 197 * State for PDs/PTs and per VM BOs which are not at the location they should 198 * be. 199 */ 200 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) 201 { 202 struct amdgpu_vm *vm = vm_bo->vm; 203 struct amdgpu_bo *bo = vm_bo->bo; 204 205 vm_bo->moved = true; 206 if (bo->tbo.type == ttm_bo_type_kernel) 207 list_move(&vm_bo->vm_status, &vm->evicted); 208 else 209 list_move_tail(&vm_bo->vm_status, &vm->evicted); 210 } 211 212 /** 213 * amdgpu_vm_bo_relocated - vm_bo is reloacted 214 * 215 * @vm_bo: vm_bo which is relocated 216 * 217 * State for PDs/PTs which needs to update their parent PD. 218 */ 219 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) 220 { 221 list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); 222 } 223 224 /** 225 * amdgpu_vm_bo_moved - vm_bo is moved 226 * 227 * @vm_bo: vm_bo which is moved 228 * 229 * State for per VM BOs which are moved, but that change is not yet reflected 230 * in the page tables. 231 */ 232 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) 233 { 234 list_move(&vm_bo->vm_status, &vm_bo->vm->moved); 235 } 236 237 /** 238 * amdgpu_vm_bo_idle - vm_bo is idle 239 * 240 * @vm_bo: vm_bo which is now idle 241 * 242 * State for PDs/PTs and per VM BOs which have gone through the state machine 243 * and are now idle. 244 */ 245 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) 246 { 247 list_move(&vm_bo->vm_status, &vm_bo->vm->idle); 248 vm_bo->moved = false; 249 } 250 251 /** 252 * amdgpu_vm_bo_invalidated - vm_bo is invalidated 253 * 254 * @vm_bo: vm_bo which is now invalidated 255 * 256 * State for normal BOs which are invalidated and that change not yet reflected 257 * in the PTs. 258 */ 259 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) 260 { 261 spin_lock(&vm_bo->vm->invalidated_lock); 262 list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); 263 spin_unlock(&vm_bo->vm->invalidated_lock); 264 } 265 266 /** 267 * amdgpu_vm_bo_done - vm_bo is done 268 * 269 * @vm_bo: vm_bo which is now done 270 * 271 * State for normal BOs which are invalidated and that change has been updated 272 * in the PTs. 273 */ 274 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo) 275 { 276 spin_lock(&vm_bo->vm->invalidated_lock); 277 list_del_init(&vm_bo->vm_status); 278 spin_unlock(&vm_bo->vm->invalidated_lock); 279 } 280 281 /** 282 * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm 283 * 284 * @base: base structure for tracking BO usage in a VM 285 * @vm: vm to which bo is to be added 286 * @bo: amdgpu buffer object 287 * 288 * Initialize a bo_va_base structure and add it to the appropriate lists 289 * 290 */ 291 static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, 292 struct amdgpu_vm *vm, 293 struct amdgpu_bo *bo) 294 { 295 base->vm = vm; 296 base->bo = bo; 297 base->next = NULL; 298 INIT_LIST_HEAD(&base->vm_status); 299 300 if (!bo) 301 return; 302 base->next = bo->vm_bo; 303 bo->vm_bo = base; 304 305 if (bo->tbo.base.resv != vm->root.base.bo->tbo.base.resv) 306 return; 307 308 vm->bulk_moveable = false; 309 if (bo->tbo.type == ttm_bo_type_kernel && bo->parent) 310 amdgpu_vm_bo_relocated(base); 311 else 312 amdgpu_vm_bo_idle(base); 313 314 if (bo->preferred_domains & 315 amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type)) 316 return; 317 318 /* 319 * we checked all the prerequisites, but it looks like this per vm bo 320 * is currently evicted. add the bo to the evicted list to make sure it 321 * is validated on next vm use to avoid fault. 322 * */ 323 amdgpu_vm_bo_evicted(base); 324 } 325 326 /** 327 * amdgpu_vm_pt_parent - get the parent page directory 328 * 329 * @pt: child page table 330 * 331 * Helper to get the parent entry for the child page table. NULL if we are at 332 * the root page directory. 333 */ 334 static struct amdgpu_vm_pt *amdgpu_vm_pt_parent(struct amdgpu_vm_pt *pt) 335 { 336 struct amdgpu_bo *parent = pt->base.bo->parent; 337 338 if (!parent) 339 return NULL; 340 341 return container_of(parent->vm_bo, struct amdgpu_vm_pt, base); 342 } 343 344 /** 345 * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt 346 */ 347 struct amdgpu_vm_pt_cursor { 348 uint64_t pfn; 349 struct amdgpu_vm_pt *parent; 350 struct amdgpu_vm_pt *entry; 351 unsigned level; 352 }; 353 354 /** 355 * amdgpu_vm_pt_start - start PD/PT walk 356 * 357 * @adev: amdgpu_device pointer 358 * @vm: amdgpu_vm structure 359 * @start: start address of the walk 360 * @cursor: state to initialize 361 * 362 * Initialize a amdgpu_vm_pt_cursor to start a walk. 363 */ 364 static void amdgpu_vm_pt_start(struct amdgpu_device *adev, 365 struct amdgpu_vm *vm, uint64_t start, 366 struct amdgpu_vm_pt_cursor *cursor) 367 { 368 cursor->pfn = start; 369 cursor->parent = NULL; 370 cursor->entry = &vm->root; 371 cursor->level = adev->vm_manager.root_level; 372 } 373 374 /** 375 * amdgpu_vm_pt_descendant - go to child node 376 * 377 * @adev: amdgpu_device pointer 378 * @cursor: current state 379 * 380 * Walk to the child node of the current node. 381 * Returns: 382 * True if the walk was possible, false otherwise. 383 */ 384 static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev, 385 struct amdgpu_vm_pt_cursor *cursor) 386 { 387 unsigned mask, shift, idx; 388 389 if (!cursor->entry->entries) 390 return false; 391 392 BUG_ON(!cursor->entry->base.bo); 393 mask = amdgpu_vm_entries_mask(adev, cursor->level); 394 shift = amdgpu_vm_level_shift(adev, cursor->level); 395 396 ++cursor->level; 397 idx = (cursor->pfn >> shift) & mask; 398 cursor->parent = cursor->entry; 399 cursor->entry = &cursor->entry->entries[idx]; 400 return true; 401 } 402 403 /** 404 * amdgpu_vm_pt_sibling - go to sibling node 405 * 406 * @adev: amdgpu_device pointer 407 * @cursor: current state 408 * 409 * Walk to the sibling node of the current node. 410 * Returns: 411 * True if the walk was possible, false otherwise. 412 */ 413 static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev, 414 struct amdgpu_vm_pt_cursor *cursor) 415 { 416 unsigned shift, num_entries; 417 418 /* Root doesn't have a sibling */ 419 if (!cursor->parent) 420 return false; 421 422 /* Go to our parents and see if we got a sibling */ 423 shift = amdgpu_vm_level_shift(adev, cursor->level - 1); 424 num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1); 425 426 if (cursor->entry == &cursor->parent->entries[num_entries - 1]) 427 return false; 428 429 cursor->pfn += 1ULL << shift; 430 cursor->pfn &= ~((1ULL << shift) - 1); 431 ++cursor->entry; 432 return true; 433 } 434 435 /** 436 * amdgpu_vm_pt_ancestor - go to parent node 437 * 438 * @cursor: current state 439 * 440 * Walk to the parent node of the current node. 441 * Returns: 442 * True if the walk was possible, false otherwise. 443 */ 444 static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor) 445 { 446 if (!cursor->parent) 447 return false; 448 449 --cursor->level; 450 cursor->entry = cursor->parent; 451 cursor->parent = amdgpu_vm_pt_parent(cursor->parent); 452 return true; 453 } 454 455 /** 456 * amdgpu_vm_pt_next - get next PD/PT in hieratchy 457 * 458 * @adev: amdgpu_device pointer 459 * @cursor: current state 460 * 461 * Walk the PD/PT tree to the next node. 462 */ 463 static void amdgpu_vm_pt_next(struct amdgpu_device *adev, 464 struct amdgpu_vm_pt_cursor *cursor) 465 { 466 /* First try a newborn child */ 467 if (amdgpu_vm_pt_descendant(adev, cursor)) 468 return; 469 470 /* If that didn't worked try to find a sibling */ 471 while (!amdgpu_vm_pt_sibling(adev, cursor)) { 472 /* No sibling, go to our parents and grandparents */ 473 if (!amdgpu_vm_pt_ancestor(cursor)) { 474 cursor->pfn = ~0ll; 475 return; 476 } 477 } 478 } 479 480 /** 481 * amdgpu_vm_pt_first_dfs - start a deep first search 482 * 483 * @adev: amdgpu_device structure 484 * @vm: amdgpu_vm structure 485 * @cursor: state to initialize 486 * 487 * Starts a deep first traversal of the PD/PT tree. 488 */ 489 static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev, 490 struct amdgpu_vm *vm, 491 struct amdgpu_vm_pt_cursor *start, 492 struct amdgpu_vm_pt_cursor *cursor) 493 { 494 if (start) 495 *cursor = *start; 496 else 497 amdgpu_vm_pt_start(adev, vm, 0, cursor); 498 while (amdgpu_vm_pt_descendant(adev, cursor)); 499 } 500 501 /** 502 * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue 503 * 504 * @start: starting point for the search 505 * @entry: current entry 506 * 507 * Returns: 508 * True when the search should continue, false otherwise. 509 */ 510 static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start, 511 struct amdgpu_vm_pt *entry) 512 { 513 return entry && (!start || entry != start->entry); 514 } 515 516 /** 517 * amdgpu_vm_pt_next_dfs - get the next node for a deep first search 518 * 519 * @adev: amdgpu_device structure 520 * @cursor: current state 521 * 522 * Move the cursor to the next node in a deep first search. 523 */ 524 static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev, 525 struct amdgpu_vm_pt_cursor *cursor) 526 { 527 if (!cursor->entry) 528 return; 529 530 if (!cursor->parent) 531 cursor->entry = NULL; 532 else if (amdgpu_vm_pt_sibling(adev, cursor)) 533 while (amdgpu_vm_pt_descendant(adev, cursor)); 534 else 535 amdgpu_vm_pt_ancestor(cursor); 536 } 537 538 /** 539 * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs 540 */ 541 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \ 542 for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \ 543 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\ 544 amdgpu_vm_pt_continue_dfs((start), (entry)); \ 545 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor))) 546 547 /** 548 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list 549 * 550 * @vm: vm providing the BOs 551 * @validated: head of validation list 552 * @entry: entry to add 553 * 554 * Add the page directory to the list of BOs to 555 * validate for command submission. 556 */ 557 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, 558 struct list_head *validated, 559 struct amdgpu_bo_list_entry *entry) 560 { 561 entry->priority = 0; 562 entry->tv.bo = &vm->root.base.bo->tbo; 563 /* One for the VM updates, one for TTM and one for the CS job */ 564 entry->tv.num_shared = 3; 565 entry->user_pages = NULL; 566 list_add(&entry->tv.head, validated); 567 } 568 569 void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo) 570 { 571 struct amdgpu_bo *abo; 572 struct amdgpu_vm_bo_base *bo_base; 573 574 if (!amdgpu_bo_is_amdgpu_bo(bo)) 575 return; 576 577 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) 578 return; 579 580 abo = ttm_to_amdgpu_bo(bo); 581 if (!abo->parent) 582 return; 583 for (bo_base = abo->vm_bo; bo_base; bo_base = bo_base->next) { 584 struct amdgpu_vm *vm = bo_base->vm; 585 586 if (abo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) 587 vm->bulk_moveable = false; 588 } 589 590 } 591 /** 592 * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU 593 * 594 * @adev: amdgpu device pointer 595 * @vm: vm providing the BOs 596 * 597 * Move all BOs to the end of LRU and remember their positions to put them 598 * together. 599 */ 600 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, 601 struct amdgpu_vm *vm) 602 { 603 struct ttm_bo_global *glob = adev->mman.bdev.glob; 604 struct amdgpu_vm_bo_base *bo_base; 605 606 if (vm->bulk_moveable) { 607 spin_lock(&glob->lru_lock); 608 ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move); 609 spin_unlock(&glob->lru_lock); 610 return; 611 } 612 613 memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move)); 614 615 spin_lock(&glob->lru_lock); 616 list_for_each_entry(bo_base, &vm->idle, vm_status) { 617 struct amdgpu_bo *bo = bo_base->bo; 618 619 if (!bo->parent) 620 continue; 621 622 ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move); 623 if (bo->shadow) 624 ttm_bo_move_to_lru_tail(&bo->shadow->tbo, 625 &vm->lru_bulk_move); 626 } 627 spin_unlock(&glob->lru_lock); 628 629 vm->bulk_moveable = true; 630 } 631 632 /** 633 * amdgpu_vm_validate_pt_bos - validate the page table BOs 634 * 635 * @adev: amdgpu device pointer 636 * @vm: vm providing the BOs 637 * @validate: callback to do the validation 638 * @param: parameter for the validation callback 639 * 640 * Validate the page table BOs on command submission if neccessary. 641 * 642 * Returns: 643 * Validation result. 644 */ 645 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, 646 int (*validate)(void *p, struct amdgpu_bo *bo), 647 void *param) 648 { 649 struct amdgpu_vm_bo_base *bo_base, *tmp; 650 int r = 0; 651 652 vm->bulk_moveable &= list_empty(&vm->evicted); 653 654 list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) { 655 struct amdgpu_bo *bo = bo_base->bo; 656 657 r = validate(param, bo); 658 if (r) 659 break; 660 661 if (bo->tbo.type != ttm_bo_type_kernel) { 662 amdgpu_vm_bo_moved(bo_base); 663 } else { 664 vm->update_funcs->map_table(bo); 665 if (bo->parent) 666 amdgpu_vm_bo_relocated(bo_base); 667 else 668 amdgpu_vm_bo_idle(bo_base); 669 } 670 } 671 672 return r; 673 } 674 675 /** 676 * amdgpu_vm_ready - check VM is ready for updates 677 * 678 * @vm: VM to check 679 * 680 * Check if all VM PDs/PTs are ready for updates 681 * 682 * Returns: 683 * True if eviction list is empty. 684 */ 685 bool amdgpu_vm_ready(struct amdgpu_vm *vm) 686 { 687 return list_empty(&vm->evicted); 688 } 689 690 /** 691 * amdgpu_vm_clear_bo - initially clear the PDs/PTs 692 * 693 * @adev: amdgpu_device pointer 694 * @vm: VM to clear BO from 695 * @bo: BO to clear 696 * 697 * Root PD needs to be reserved when calling this. 698 * 699 * Returns: 700 * 0 on success, errno otherwise. 701 */ 702 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, 703 struct amdgpu_vm *vm, 704 struct amdgpu_bo *bo) 705 { 706 struct ttm_operation_ctx ctx = { true, false }; 707 unsigned level = adev->vm_manager.root_level; 708 struct amdgpu_vm_update_params params; 709 struct amdgpu_bo *ancestor = bo; 710 unsigned entries, ats_entries; 711 uint64_t addr; 712 int r; 713 714 /* Figure out our place in the hierarchy */ 715 if (ancestor->parent) { 716 ++level; 717 while (ancestor->parent->parent) { 718 ++level; 719 ancestor = ancestor->parent; 720 } 721 } 722 723 entries = amdgpu_bo_size(bo) / 8; 724 if (!vm->pte_support_ats) { 725 ats_entries = 0; 726 727 } else if (!bo->parent) { 728 ats_entries = amdgpu_vm_num_ats_entries(adev); 729 ats_entries = min(ats_entries, entries); 730 entries -= ats_entries; 731 732 } else { 733 struct amdgpu_vm_pt *pt; 734 735 pt = container_of(ancestor->vm_bo, struct amdgpu_vm_pt, base); 736 ats_entries = amdgpu_vm_num_ats_entries(adev); 737 if ((pt - vm->root.entries) >= ats_entries) { 738 ats_entries = 0; 739 } else { 740 ats_entries = entries; 741 entries = 0; 742 } 743 } 744 745 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 746 if (r) 747 return r; 748 749 if (bo->shadow) { 750 r = ttm_bo_validate(&bo->shadow->tbo, &bo->shadow->placement, 751 &ctx); 752 if (r) 753 return r; 754 } 755 756 r = vm->update_funcs->map_table(bo); 757 if (r) 758 return r; 759 760 memset(¶ms, 0, sizeof(params)); 761 params.adev = adev; 762 params.vm = vm; 763 764 r = vm->update_funcs->prepare(¶ms, AMDGPU_FENCE_OWNER_KFD, NULL); 765 if (r) 766 return r; 767 768 addr = 0; 769 if (ats_entries) { 770 uint64_t value = 0, flags; 771 772 flags = AMDGPU_PTE_DEFAULT_ATC; 773 if (level != AMDGPU_VM_PTB) { 774 /* Handle leaf PDEs as PTEs */ 775 flags |= AMDGPU_PDE_PTE; 776 amdgpu_gmc_get_vm_pde(adev, level, &value, &flags); 777 } 778 779 r = vm->update_funcs->update(¶ms, bo, addr, 0, ats_entries, 780 value, flags); 781 if (r) 782 return r; 783 784 addr += ats_entries * 8; 785 } 786 787 if (entries) { 788 uint64_t value = 0, flags = 0; 789 790 if (adev->asic_type >= CHIP_VEGA10) { 791 if (level != AMDGPU_VM_PTB) { 792 /* Handle leaf PDEs as PTEs */ 793 flags |= AMDGPU_PDE_PTE; 794 amdgpu_gmc_get_vm_pde(adev, level, 795 &value, &flags); 796 } else { 797 /* Workaround for fault priority problem on GMC9 */ 798 flags = AMDGPU_PTE_EXECUTABLE; 799 } 800 } 801 802 r = vm->update_funcs->update(¶ms, bo, addr, 0, entries, 803 value, flags); 804 if (r) 805 return r; 806 } 807 808 return vm->update_funcs->commit(¶ms, NULL); 809 } 810 811 /** 812 * amdgpu_vm_bo_param - fill in parameters for PD/PT allocation 813 * 814 * @adev: amdgpu_device pointer 815 * @vm: requesting vm 816 * @bp: resulting BO allocation parameters 817 */ 818 static void amdgpu_vm_bo_param(struct amdgpu_device *adev, struct amdgpu_vm *vm, 819 int level, struct amdgpu_bo_param *bp) 820 { 821 memset(bp, 0, sizeof(*bp)); 822 823 bp->size = amdgpu_vm_bo_size(adev, level); 824 bp->byte_align = AMDGPU_GPU_PAGE_SIZE; 825 bp->domain = AMDGPU_GEM_DOMAIN_VRAM; 826 bp->domain = amdgpu_bo_get_preferred_pin_domain(adev, bp->domain); 827 bp->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | 828 AMDGPU_GEM_CREATE_CPU_GTT_USWC; 829 if (vm->use_cpu_for_update) 830 bp->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 831 else if (!vm->root.base.bo || vm->root.base.bo->shadow) 832 bp->flags |= AMDGPU_GEM_CREATE_SHADOW; 833 bp->type = ttm_bo_type_kernel; 834 if (vm->root.base.bo) 835 bp->resv = vm->root.base.bo->tbo.base.resv; 836 } 837 838 /** 839 * amdgpu_vm_alloc_pts - Allocate a specific page table 840 * 841 * @adev: amdgpu_device pointer 842 * @vm: VM to allocate page tables for 843 * @cursor: Which page table to allocate 844 * 845 * Make sure a specific page table or directory is allocated. 846 * 847 * Returns: 848 * 1 if page table needed to be allocated, 0 if page table was already 849 * allocated, negative errno if an error occurred. 850 */ 851 static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, 852 struct amdgpu_vm *vm, 853 struct amdgpu_vm_pt_cursor *cursor) 854 { 855 struct amdgpu_vm_pt *entry = cursor->entry; 856 struct amdgpu_bo_param bp; 857 struct amdgpu_bo *pt; 858 int r; 859 860 if (cursor->level < AMDGPU_VM_PTB && !entry->entries) { 861 unsigned num_entries; 862 863 num_entries = amdgpu_vm_num_entries(adev, cursor->level); 864 entry->entries = kvmalloc_array(num_entries, 865 sizeof(*entry->entries), 866 GFP_KERNEL | __GFP_ZERO); 867 if (!entry->entries) 868 return -ENOMEM; 869 } 870 871 if (entry->base.bo) 872 return 0; 873 874 amdgpu_vm_bo_param(adev, vm, cursor->level, &bp); 875 876 r = amdgpu_bo_create(adev, &bp, &pt); 877 if (r) 878 return r; 879 880 /* Keep a reference to the root directory to avoid 881 * freeing them up in the wrong order. 882 */ 883 pt->parent = amdgpu_bo_ref(cursor->parent->base.bo); 884 amdgpu_vm_bo_base_init(&entry->base, vm, pt); 885 886 r = amdgpu_vm_clear_bo(adev, vm, pt); 887 if (r) 888 goto error_free_pt; 889 890 return 0; 891 892 error_free_pt: 893 amdgpu_bo_unref(&pt->shadow); 894 amdgpu_bo_unref(&pt); 895 return r; 896 } 897 898 /** 899 * amdgpu_vm_free_table - fre one PD/PT 900 * 901 * @entry: PDE to free 902 */ 903 static void amdgpu_vm_free_table(struct amdgpu_vm_pt *entry) 904 { 905 if (entry->base.bo) { 906 entry->base.bo->vm_bo = NULL; 907 list_del(&entry->base.vm_status); 908 amdgpu_bo_unref(&entry->base.bo->shadow); 909 amdgpu_bo_unref(&entry->base.bo); 910 } 911 kvfree(entry->entries); 912 entry->entries = NULL; 913 } 914 915 /** 916 * amdgpu_vm_free_pts - free PD/PT levels 917 * 918 * @adev: amdgpu device structure 919 * @vm: amdgpu vm structure 920 * @start: optional cursor where to start freeing PDs/PTs 921 * 922 * Free the page directory or page table level and all sub levels. 923 */ 924 static void amdgpu_vm_free_pts(struct amdgpu_device *adev, 925 struct amdgpu_vm *vm, 926 struct amdgpu_vm_pt_cursor *start) 927 { 928 struct amdgpu_vm_pt_cursor cursor; 929 struct amdgpu_vm_pt *entry; 930 931 vm->bulk_moveable = false; 932 933 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) 934 amdgpu_vm_free_table(entry); 935 936 if (start) 937 amdgpu_vm_free_table(start->entry); 938 } 939 940 /** 941 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug 942 * 943 * @adev: amdgpu_device pointer 944 */ 945 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev) 946 { 947 const struct amdgpu_ip_block *ip_block; 948 bool has_compute_vm_bug; 949 struct amdgpu_ring *ring; 950 int i; 951 952 has_compute_vm_bug = false; 953 954 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); 955 if (ip_block) { 956 /* Compute has a VM bug for GFX version < 7. 957 Compute has a VM bug for GFX 8 MEC firmware version < 673.*/ 958 if (ip_block->version->major <= 7) 959 has_compute_vm_bug = true; 960 else if (ip_block->version->major == 8) 961 if (adev->gfx.mec_fw_version < 673) 962 has_compute_vm_bug = true; 963 } 964 965 for (i = 0; i < adev->num_rings; i++) { 966 ring = adev->rings[i]; 967 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 968 /* only compute rings */ 969 ring->has_compute_vm_bug = has_compute_vm_bug; 970 else 971 ring->has_compute_vm_bug = false; 972 } 973 } 974 975 /** 976 * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job. 977 * 978 * @ring: ring on which the job will be submitted 979 * @job: job to submit 980 * 981 * Returns: 982 * True if sync is needed. 983 */ 984 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring, 985 struct amdgpu_job *job) 986 { 987 struct amdgpu_device *adev = ring->adev; 988 unsigned vmhub = ring->funcs->vmhub; 989 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 990 struct amdgpu_vmid *id; 991 bool gds_switch_needed; 992 bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug; 993 994 if (job->vmid == 0) 995 return false; 996 id = &id_mgr->ids[job->vmid]; 997 gds_switch_needed = ring->funcs->emit_gds_switch && ( 998 id->gds_base != job->gds_base || 999 id->gds_size != job->gds_size || 1000 id->gws_base != job->gws_base || 1001 id->gws_size != job->gws_size || 1002 id->oa_base != job->oa_base || 1003 id->oa_size != job->oa_size); 1004 1005 if (amdgpu_vmid_had_gpu_reset(adev, id)) 1006 return true; 1007 1008 return vm_flush_needed || gds_switch_needed; 1009 } 1010 1011 /** 1012 * amdgpu_vm_flush - hardware flush the vm 1013 * 1014 * @ring: ring to use for flush 1015 * @job: related job 1016 * @need_pipe_sync: is pipe sync needed 1017 * 1018 * Emit a VM flush when it is necessary. 1019 * 1020 * Returns: 1021 * 0 on success, errno otherwise. 1022 */ 1023 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync) 1024 { 1025 struct amdgpu_device *adev = ring->adev; 1026 unsigned vmhub = ring->funcs->vmhub; 1027 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 1028 struct amdgpu_vmid *id = &id_mgr->ids[job->vmid]; 1029 bool gds_switch_needed = ring->funcs->emit_gds_switch && ( 1030 id->gds_base != job->gds_base || 1031 id->gds_size != job->gds_size || 1032 id->gws_base != job->gws_base || 1033 id->gws_size != job->gws_size || 1034 id->oa_base != job->oa_base || 1035 id->oa_size != job->oa_size); 1036 bool vm_flush_needed = job->vm_needs_flush; 1037 bool pasid_mapping_needed = id->pasid != job->pasid || 1038 !id->pasid_mapping || 1039 !dma_fence_is_signaled(id->pasid_mapping); 1040 struct dma_fence *fence = NULL; 1041 unsigned patch_offset = 0; 1042 int r; 1043 1044 if (amdgpu_vmid_had_gpu_reset(adev, id)) { 1045 gds_switch_needed = true; 1046 vm_flush_needed = true; 1047 pasid_mapping_needed = true; 1048 } 1049 1050 gds_switch_needed &= !!ring->funcs->emit_gds_switch; 1051 vm_flush_needed &= !!ring->funcs->emit_vm_flush && 1052 job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET; 1053 pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping && 1054 ring->funcs->emit_wreg; 1055 1056 if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync) 1057 return 0; 1058 1059 if (ring->funcs->init_cond_exec) 1060 patch_offset = amdgpu_ring_init_cond_exec(ring); 1061 1062 if (need_pipe_sync) 1063 amdgpu_ring_emit_pipeline_sync(ring); 1064 1065 if (vm_flush_needed) { 1066 trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr); 1067 amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr); 1068 } 1069 1070 if (pasid_mapping_needed) 1071 amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid); 1072 1073 if (vm_flush_needed || pasid_mapping_needed) { 1074 r = amdgpu_fence_emit(ring, &fence, 0); 1075 if (r) 1076 return r; 1077 } 1078 1079 if (vm_flush_needed) { 1080 mutex_lock(&id_mgr->lock); 1081 dma_fence_put(id->last_flush); 1082 id->last_flush = dma_fence_get(fence); 1083 id->current_gpu_reset_count = 1084 atomic_read(&adev->gpu_reset_counter); 1085 mutex_unlock(&id_mgr->lock); 1086 } 1087 1088 if (pasid_mapping_needed) { 1089 id->pasid = job->pasid; 1090 dma_fence_put(id->pasid_mapping); 1091 id->pasid_mapping = dma_fence_get(fence); 1092 } 1093 dma_fence_put(fence); 1094 1095 if (ring->funcs->emit_gds_switch && gds_switch_needed) { 1096 id->gds_base = job->gds_base; 1097 id->gds_size = job->gds_size; 1098 id->gws_base = job->gws_base; 1099 id->gws_size = job->gws_size; 1100 id->oa_base = job->oa_base; 1101 id->oa_size = job->oa_size; 1102 amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base, 1103 job->gds_size, job->gws_base, 1104 job->gws_size, job->oa_base, 1105 job->oa_size); 1106 } 1107 1108 if (ring->funcs->patch_cond_exec) 1109 amdgpu_ring_patch_cond_exec(ring, patch_offset); 1110 1111 /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */ 1112 if (ring->funcs->emit_switch_buffer) { 1113 amdgpu_ring_emit_switch_buffer(ring); 1114 amdgpu_ring_emit_switch_buffer(ring); 1115 } 1116 return 0; 1117 } 1118 1119 /** 1120 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo 1121 * 1122 * @vm: requested vm 1123 * @bo: requested buffer object 1124 * 1125 * Find @bo inside the requested vm. 1126 * Search inside the @bos vm list for the requested vm 1127 * Returns the found bo_va or NULL if none is found 1128 * 1129 * Object has to be reserved! 1130 * 1131 * Returns: 1132 * Found bo_va or NULL. 1133 */ 1134 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, 1135 struct amdgpu_bo *bo) 1136 { 1137 struct amdgpu_vm_bo_base *base; 1138 1139 for (base = bo->vm_bo; base; base = base->next) { 1140 if (base->vm != vm) 1141 continue; 1142 1143 return container_of(base, struct amdgpu_bo_va, base); 1144 } 1145 return NULL; 1146 } 1147 1148 /** 1149 * amdgpu_vm_map_gart - Resolve gart mapping of addr 1150 * 1151 * @pages_addr: optional DMA address to use for lookup 1152 * @addr: the unmapped addr 1153 * 1154 * Look up the physical address of the page that the pte resolves 1155 * to. 1156 * 1157 * Returns: 1158 * The pointer for the page table entry. 1159 */ 1160 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr) 1161 { 1162 uint64_t result; 1163 1164 /* page table offset */ 1165 result = pages_addr[addr >> PAGE_SHIFT]; 1166 1167 /* in case cpu page size != gpu page size*/ 1168 result |= addr & (~PAGE_MASK); 1169 1170 result &= 0xFFFFFFFFFFFFF000ULL; 1171 1172 return result; 1173 } 1174 1175 /* 1176 * amdgpu_vm_update_pde - update a single level in the hierarchy 1177 * 1178 * @param: parameters for the update 1179 * @vm: requested vm 1180 * @entry: entry to update 1181 * 1182 * Makes sure the requested entry in parent is up to date. 1183 */ 1184 static int amdgpu_vm_update_pde(struct amdgpu_vm_update_params *params, 1185 struct amdgpu_vm *vm, 1186 struct amdgpu_vm_pt *entry) 1187 { 1188 struct amdgpu_vm_pt *parent = amdgpu_vm_pt_parent(entry); 1189 struct amdgpu_bo *bo = parent->base.bo, *pbo; 1190 uint64_t pde, pt, flags; 1191 unsigned level; 1192 1193 for (level = 0, pbo = bo->parent; pbo; ++level) 1194 pbo = pbo->parent; 1195 1196 level += params->adev->vm_manager.root_level; 1197 amdgpu_gmc_get_pde_for_bo(entry->base.bo, level, &pt, &flags); 1198 pde = (entry - parent->entries) * 8; 1199 return vm->update_funcs->update(params, bo, pde, pt, 1, 0, flags); 1200 } 1201 1202 /* 1203 * amdgpu_vm_invalidate_pds - mark all PDs as invalid 1204 * 1205 * @adev: amdgpu_device pointer 1206 * @vm: related vm 1207 * 1208 * Mark all PD level as invalid after an error. 1209 */ 1210 static void amdgpu_vm_invalidate_pds(struct amdgpu_device *adev, 1211 struct amdgpu_vm *vm) 1212 { 1213 struct amdgpu_vm_pt_cursor cursor; 1214 struct amdgpu_vm_pt *entry; 1215 1216 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry) 1217 if (entry->base.bo && !entry->base.moved) 1218 amdgpu_vm_bo_relocated(&entry->base); 1219 } 1220 1221 /* 1222 * amdgpu_vm_update_directories - make sure that all directories are valid 1223 * 1224 * @adev: amdgpu_device pointer 1225 * @vm: requested vm 1226 * 1227 * Makes sure all directories are up to date. 1228 * 1229 * Returns: 1230 * 0 for success, error for failure. 1231 */ 1232 int amdgpu_vm_update_directories(struct amdgpu_device *adev, 1233 struct amdgpu_vm *vm) 1234 { 1235 struct amdgpu_vm_update_params params; 1236 int r; 1237 1238 if (list_empty(&vm->relocated)) 1239 return 0; 1240 1241 memset(¶ms, 0, sizeof(params)); 1242 params.adev = adev; 1243 params.vm = vm; 1244 1245 r = vm->update_funcs->prepare(¶ms, AMDGPU_FENCE_OWNER_VM, NULL); 1246 if (r) 1247 return r; 1248 1249 while (!list_empty(&vm->relocated)) { 1250 struct amdgpu_vm_pt *entry; 1251 1252 entry = list_first_entry(&vm->relocated, struct amdgpu_vm_pt, 1253 base.vm_status); 1254 amdgpu_vm_bo_idle(&entry->base); 1255 1256 r = amdgpu_vm_update_pde(¶ms, vm, entry); 1257 if (r) 1258 goto error; 1259 } 1260 1261 r = vm->update_funcs->commit(¶ms, &vm->last_update); 1262 if (r) 1263 goto error; 1264 return 0; 1265 1266 error: 1267 amdgpu_vm_invalidate_pds(adev, vm); 1268 return r; 1269 } 1270 1271 /** 1272 * amdgpu_vm_update_flags - figure out flags for PTE updates 1273 * 1274 * Make sure to set the right flags for the PTEs at the desired level. 1275 */ 1276 static void amdgpu_vm_update_flags(struct amdgpu_vm_update_params *params, 1277 struct amdgpu_bo *bo, unsigned level, 1278 uint64_t pe, uint64_t addr, 1279 unsigned count, uint32_t incr, 1280 uint64_t flags) 1281 1282 { 1283 if (level != AMDGPU_VM_PTB) { 1284 flags |= AMDGPU_PDE_PTE; 1285 amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags); 1286 1287 } else if (params->adev->asic_type >= CHIP_VEGA10 && 1288 !(flags & AMDGPU_PTE_VALID) && 1289 !(flags & AMDGPU_PTE_PRT)) { 1290 1291 /* Workaround for fault priority problem on GMC9 */ 1292 flags |= AMDGPU_PTE_EXECUTABLE; 1293 } 1294 1295 params->vm->update_funcs->update(params, bo, pe, addr, count, incr, 1296 flags); 1297 } 1298 1299 /** 1300 * amdgpu_vm_fragment - get fragment for PTEs 1301 * 1302 * @params: see amdgpu_vm_update_params definition 1303 * @start: first PTE to handle 1304 * @end: last PTE to handle 1305 * @flags: hw mapping flags 1306 * @frag: resulting fragment size 1307 * @frag_end: end of this fragment 1308 * 1309 * Returns the first possible fragment for the start and end address. 1310 */ 1311 static void amdgpu_vm_fragment(struct amdgpu_vm_update_params *params, 1312 uint64_t start, uint64_t end, uint64_t flags, 1313 unsigned int *frag, uint64_t *frag_end) 1314 { 1315 /** 1316 * The MC L1 TLB supports variable sized pages, based on a fragment 1317 * field in the PTE. When this field is set to a non-zero value, page 1318 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE 1319 * flags are considered valid for all PTEs within the fragment range 1320 * and corresponding mappings are assumed to be physically contiguous. 1321 * 1322 * The L1 TLB can store a single PTE for the whole fragment, 1323 * significantly increasing the space available for translation 1324 * caching. This leads to large improvements in throughput when the 1325 * TLB is under pressure. 1326 * 1327 * The L2 TLB distributes small and large fragments into two 1328 * asymmetric partitions. The large fragment cache is significantly 1329 * larger. Thus, we try to use large fragments wherever possible. 1330 * Userspace can support this by aligning virtual base address and 1331 * allocation size to the fragment size. 1332 * 1333 * Starting with Vega10 the fragment size only controls the L1. The L2 1334 * is now directly feed with small/huge/giant pages from the walker. 1335 */ 1336 unsigned max_frag; 1337 1338 if (params->adev->asic_type < CHIP_VEGA10) 1339 max_frag = params->adev->vm_manager.fragment_size; 1340 else 1341 max_frag = 31; 1342 1343 /* system pages are non continuously */ 1344 if (params->pages_addr) { 1345 *frag = 0; 1346 *frag_end = end; 1347 return; 1348 } 1349 1350 /* This intentionally wraps around if no bit is set */ 1351 *frag = min((unsigned)ffs(start) - 1, (unsigned)fls64(end - start) - 1); 1352 if (*frag >= max_frag) { 1353 *frag = max_frag; 1354 *frag_end = end & ~((1ULL << max_frag) - 1); 1355 } else { 1356 *frag_end = start + (1 << *frag); 1357 } 1358 } 1359 1360 /** 1361 * amdgpu_vm_update_ptes - make sure that page tables are valid 1362 * 1363 * @params: see amdgpu_vm_update_params definition 1364 * @start: start of GPU address range 1365 * @end: end of GPU address range 1366 * @dst: destination address to map to, the next dst inside the function 1367 * @flags: mapping flags 1368 * 1369 * Update the page tables in the range @start - @end. 1370 * 1371 * Returns: 1372 * 0 for success, -EINVAL for failure. 1373 */ 1374 static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params, 1375 uint64_t start, uint64_t end, 1376 uint64_t dst, uint64_t flags) 1377 { 1378 struct amdgpu_device *adev = params->adev; 1379 struct amdgpu_vm_pt_cursor cursor; 1380 uint64_t frag_start = start, frag_end; 1381 unsigned int frag; 1382 int r; 1383 1384 /* figure out the initial fragment */ 1385 amdgpu_vm_fragment(params, frag_start, end, flags, &frag, &frag_end); 1386 1387 /* walk over the address space and update the PTs */ 1388 amdgpu_vm_pt_start(adev, params->vm, start, &cursor); 1389 while (cursor.pfn < end) { 1390 unsigned shift, parent_shift, mask; 1391 uint64_t incr, entry_end, pe_start; 1392 struct amdgpu_bo *pt; 1393 1394 r = amdgpu_vm_alloc_pts(params->adev, params->vm, &cursor); 1395 if (r) 1396 return r; 1397 1398 pt = cursor.entry->base.bo; 1399 1400 /* The root level can't be a huge page */ 1401 if (cursor.level == adev->vm_manager.root_level) { 1402 if (!amdgpu_vm_pt_descendant(adev, &cursor)) 1403 return -ENOENT; 1404 continue; 1405 } 1406 1407 shift = amdgpu_vm_level_shift(adev, cursor.level); 1408 parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1); 1409 if (adev->asic_type < CHIP_VEGA10 && 1410 (flags & AMDGPU_PTE_VALID)) { 1411 /* No huge page support before GMC v9 */ 1412 if (cursor.level != AMDGPU_VM_PTB) { 1413 if (!amdgpu_vm_pt_descendant(adev, &cursor)) 1414 return -ENOENT; 1415 continue; 1416 } 1417 } else if (frag < shift) { 1418 /* We can't use this level when the fragment size is 1419 * smaller than the address shift. Go to the next 1420 * child entry and try again. 1421 */ 1422 if (!amdgpu_vm_pt_descendant(adev, &cursor)) 1423 return -ENOENT; 1424 continue; 1425 } else if (frag >= parent_shift && 1426 cursor.level - 1 != adev->vm_manager.root_level) { 1427 /* If the fragment size is even larger than the parent 1428 * shift we should go up one level and check it again 1429 * unless one level up is the root level. 1430 */ 1431 if (!amdgpu_vm_pt_ancestor(&cursor)) 1432 return -ENOENT; 1433 continue; 1434 } 1435 1436 /* Looks good so far, calculate parameters for the update */ 1437 incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift; 1438 mask = amdgpu_vm_entries_mask(adev, cursor.level); 1439 pe_start = ((cursor.pfn >> shift) & mask) * 8; 1440 entry_end = (uint64_t)(mask + 1) << shift; 1441 entry_end += cursor.pfn & ~(entry_end - 1); 1442 entry_end = min(entry_end, end); 1443 1444 do { 1445 uint64_t upd_end = min(entry_end, frag_end); 1446 unsigned nptes = (upd_end - frag_start) >> shift; 1447 1448 amdgpu_vm_update_flags(params, pt, cursor.level, 1449 pe_start, dst, nptes, incr, 1450 flags | AMDGPU_PTE_FRAG(frag)); 1451 1452 pe_start += nptes * 8; 1453 dst += (uint64_t)nptes * AMDGPU_GPU_PAGE_SIZE << shift; 1454 1455 frag_start = upd_end; 1456 if (frag_start >= frag_end) { 1457 /* figure out the next fragment */ 1458 amdgpu_vm_fragment(params, frag_start, end, 1459 flags, &frag, &frag_end); 1460 if (frag < shift) 1461 break; 1462 } 1463 } while (frag_start < entry_end); 1464 1465 if (amdgpu_vm_pt_descendant(adev, &cursor)) { 1466 /* Free all child entries */ 1467 while (cursor.pfn < frag_start) { 1468 amdgpu_vm_free_pts(adev, params->vm, &cursor); 1469 amdgpu_vm_pt_next(adev, &cursor); 1470 } 1471 1472 } else if (frag >= shift) { 1473 /* or just move on to the next on the same level. */ 1474 amdgpu_vm_pt_next(adev, &cursor); 1475 } 1476 } 1477 1478 return 0; 1479 } 1480 1481 /** 1482 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table 1483 * 1484 * @adev: amdgpu_device pointer 1485 * @exclusive: fence we need to sync to 1486 * @pages_addr: DMA addresses to use for mapping 1487 * @vm: requested vm 1488 * @start: start of mapped range 1489 * @last: last mapped entry 1490 * @flags: flags for the entries 1491 * @addr: addr to set the area to 1492 * @fence: optional resulting fence 1493 * 1494 * Fill in the page table entries between @start and @last. 1495 * 1496 * Returns: 1497 * 0 for success, -EINVAL for failure. 1498 */ 1499 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, 1500 struct dma_fence *exclusive, 1501 dma_addr_t *pages_addr, 1502 struct amdgpu_vm *vm, 1503 uint64_t start, uint64_t last, 1504 uint64_t flags, uint64_t addr, 1505 struct dma_fence **fence) 1506 { 1507 struct amdgpu_vm_update_params params; 1508 void *owner = AMDGPU_FENCE_OWNER_VM; 1509 int r; 1510 1511 memset(¶ms, 0, sizeof(params)); 1512 params.adev = adev; 1513 params.vm = vm; 1514 params.pages_addr = pages_addr; 1515 1516 /* sync to everything except eviction fences on unmapping */ 1517 if (!(flags & AMDGPU_PTE_VALID)) 1518 owner = AMDGPU_FENCE_OWNER_KFD; 1519 1520 r = vm->update_funcs->prepare(¶ms, owner, exclusive); 1521 if (r) 1522 return r; 1523 1524 r = amdgpu_vm_update_ptes(¶ms, start, last + 1, addr, flags); 1525 if (r) 1526 return r; 1527 1528 return vm->update_funcs->commit(¶ms, fence); 1529 } 1530 1531 /** 1532 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks 1533 * 1534 * @adev: amdgpu_device pointer 1535 * @exclusive: fence we need to sync to 1536 * @pages_addr: DMA addresses to use for mapping 1537 * @vm: requested vm 1538 * @mapping: mapped range and flags to use for the update 1539 * @flags: HW flags for the mapping 1540 * @bo_adev: amdgpu_device pointer that bo actually been allocated 1541 * @nodes: array of drm_mm_nodes with the MC addresses 1542 * @fence: optional resulting fence 1543 * 1544 * Split the mapping into smaller chunks so that each update fits 1545 * into a SDMA IB. 1546 * 1547 * Returns: 1548 * 0 for success, -EINVAL for failure. 1549 */ 1550 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, 1551 struct dma_fence *exclusive, 1552 dma_addr_t *pages_addr, 1553 struct amdgpu_vm *vm, 1554 struct amdgpu_bo_va_mapping *mapping, 1555 uint64_t flags, 1556 struct amdgpu_device *bo_adev, 1557 struct drm_mm_node *nodes, 1558 struct dma_fence **fence) 1559 { 1560 unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size; 1561 uint64_t pfn, start = mapping->start; 1562 int r; 1563 1564 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here 1565 * but in case of something, we filter the flags in first place 1566 */ 1567 if (!(mapping->flags & AMDGPU_PTE_READABLE)) 1568 flags &= ~AMDGPU_PTE_READABLE; 1569 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) 1570 flags &= ~AMDGPU_PTE_WRITEABLE; 1571 1572 flags &= ~AMDGPU_PTE_EXECUTABLE; 1573 flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; 1574 1575 if (adev->asic_type >= CHIP_NAVI10) { 1576 flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK; 1577 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK); 1578 } else { 1579 flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; 1580 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK); 1581 } 1582 1583 if ((mapping->flags & AMDGPU_PTE_PRT) && 1584 (adev->asic_type >= CHIP_VEGA10)) { 1585 flags |= AMDGPU_PTE_PRT; 1586 if (adev->asic_type >= CHIP_NAVI10) { 1587 flags |= AMDGPU_PTE_SNOOPED; 1588 flags |= AMDGPU_PTE_LOG; 1589 flags |= AMDGPU_PTE_SYSTEM; 1590 } 1591 flags &= ~AMDGPU_PTE_VALID; 1592 } 1593 1594 trace_amdgpu_vm_bo_update(mapping); 1595 1596 pfn = mapping->offset >> PAGE_SHIFT; 1597 if (nodes) { 1598 while (pfn >= nodes->size) { 1599 pfn -= nodes->size; 1600 ++nodes; 1601 } 1602 } 1603 1604 do { 1605 dma_addr_t *dma_addr = NULL; 1606 uint64_t max_entries; 1607 uint64_t addr, last; 1608 1609 if (nodes) { 1610 addr = nodes->start << PAGE_SHIFT; 1611 max_entries = (nodes->size - pfn) * 1612 AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1613 } else { 1614 addr = 0; 1615 max_entries = S64_MAX; 1616 } 1617 1618 if (pages_addr) { 1619 uint64_t count; 1620 1621 for (count = 1; 1622 count < max_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1623 ++count) { 1624 uint64_t idx = pfn + count; 1625 1626 if (pages_addr[idx] != 1627 (pages_addr[idx - 1] + PAGE_SIZE)) 1628 break; 1629 } 1630 1631 if (count < min_linear_pages) { 1632 addr = pfn << PAGE_SHIFT; 1633 dma_addr = pages_addr; 1634 } else { 1635 addr = pages_addr[pfn]; 1636 max_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1637 } 1638 1639 } else if (flags & AMDGPU_PTE_VALID) { 1640 addr += bo_adev->vm_manager.vram_base_offset; 1641 addr += pfn << PAGE_SHIFT; 1642 } 1643 1644 last = min((uint64_t)mapping->last, start + max_entries - 1); 1645 r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm, 1646 start, last, flags, addr, 1647 fence); 1648 if (r) 1649 return r; 1650 1651 pfn += (last - start + 1) / AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1652 if (nodes && nodes->size == pfn) { 1653 pfn = 0; 1654 ++nodes; 1655 } 1656 start = last + 1; 1657 1658 } while (unlikely(start != mapping->last + 1)); 1659 1660 return 0; 1661 } 1662 1663 /** 1664 * amdgpu_vm_bo_update - update all BO mappings in the vm page table 1665 * 1666 * @adev: amdgpu_device pointer 1667 * @bo_va: requested BO and VM object 1668 * @clear: if true clear the entries 1669 * 1670 * Fill in the page table entries for @bo_va. 1671 * 1672 * Returns: 1673 * 0 for success, -EINVAL for failure. 1674 */ 1675 int amdgpu_vm_bo_update(struct amdgpu_device *adev, 1676 struct amdgpu_bo_va *bo_va, 1677 bool clear) 1678 { 1679 struct amdgpu_bo *bo = bo_va->base.bo; 1680 struct amdgpu_vm *vm = bo_va->base.vm; 1681 struct amdgpu_bo_va_mapping *mapping; 1682 dma_addr_t *pages_addr = NULL; 1683 struct ttm_mem_reg *mem; 1684 struct drm_mm_node *nodes; 1685 struct dma_fence *exclusive, **last_update; 1686 uint64_t flags; 1687 struct amdgpu_device *bo_adev = adev; 1688 int r; 1689 1690 if (clear || !bo) { 1691 mem = NULL; 1692 nodes = NULL; 1693 exclusive = NULL; 1694 } else { 1695 struct ttm_dma_tt *ttm; 1696 1697 mem = &bo->tbo.mem; 1698 nodes = mem->mm_node; 1699 if (mem->mem_type == TTM_PL_TT) { 1700 ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm); 1701 pages_addr = ttm->dma_address; 1702 } 1703 exclusive = dma_resv_get_excl(bo->tbo.base.resv); 1704 } 1705 1706 if (bo) { 1707 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem); 1708 bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); 1709 } else { 1710 flags = 0x0; 1711 } 1712 1713 if (clear || (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv)) 1714 last_update = &vm->last_update; 1715 else 1716 last_update = &bo_va->last_pt_update; 1717 1718 if (!clear && bo_va->base.moved) { 1719 bo_va->base.moved = false; 1720 list_splice_init(&bo_va->valids, &bo_va->invalids); 1721 1722 } else if (bo_va->cleared != clear) { 1723 list_splice_init(&bo_va->valids, &bo_va->invalids); 1724 } 1725 1726 list_for_each_entry(mapping, &bo_va->invalids, list) { 1727 r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm, 1728 mapping, flags, bo_adev, nodes, 1729 last_update); 1730 if (r) 1731 return r; 1732 } 1733 1734 if (vm->use_cpu_for_update) { 1735 /* Flush HDP */ 1736 mb(); 1737 amdgpu_asic_flush_hdp(adev, NULL); 1738 } 1739 1740 /* If the BO is not in its preferred location add it back to 1741 * the evicted list so that it gets validated again on the 1742 * next command submission. 1743 */ 1744 if (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) { 1745 uint32_t mem_type = bo->tbo.mem.mem_type; 1746 1747 if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type))) 1748 amdgpu_vm_bo_evicted(&bo_va->base); 1749 else 1750 amdgpu_vm_bo_idle(&bo_va->base); 1751 } else { 1752 amdgpu_vm_bo_done(&bo_va->base); 1753 } 1754 1755 list_splice_init(&bo_va->invalids, &bo_va->valids); 1756 bo_va->cleared = clear; 1757 1758 if (trace_amdgpu_vm_bo_mapping_enabled()) { 1759 list_for_each_entry(mapping, &bo_va->valids, list) 1760 trace_amdgpu_vm_bo_mapping(mapping); 1761 } 1762 1763 return 0; 1764 } 1765 1766 /** 1767 * amdgpu_vm_update_prt_state - update the global PRT state 1768 * 1769 * @adev: amdgpu_device pointer 1770 */ 1771 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev) 1772 { 1773 unsigned long flags; 1774 bool enable; 1775 1776 spin_lock_irqsave(&adev->vm_manager.prt_lock, flags); 1777 enable = !!atomic_read(&adev->vm_manager.num_prt_users); 1778 adev->gmc.gmc_funcs->set_prt(adev, enable); 1779 spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags); 1780 } 1781 1782 /** 1783 * amdgpu_vm_prt_get - add a PRT user 1784 * 1785 * @adev: amdgpu_device pointer 1786 */ 1787 static void amdgpu_vm_prt_get(struct amdgpu_device *adev) 1788 { 1789 if (!adev->gmc.gmc_funcs->set_prt) 1790 return; 1791 1792 if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1) 1793 amdgpu_vm_update_prt_state(adev); 1794 } 1795 1796 /** 1797 * amdgpu_vm_prt_put - drop a PRT user 1798 * 1799 * @adev: amdgpu_device pointer 1800 */ 1801 static void amdgpu_vm_prt_put(struct amdgpu_device *adev) 1802 { 1803 if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0) 1804 amdgpu_vm_update_prt_state(adev); 1805 } 1806 1807 /** 1808 * amdgpu_vm_prt_cb - callback for updating the PRT status 1809 * 1810 * @fence: fence for the callback 1811 * @_cb: the callback function 1812 */ 1813 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb) 1814 { 1815 struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb); 1816 1817 amdgpu_vm_prt_put(cb->adev); 1818 kfree(cb); 1819 } 1820 1821 /** 1822 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status 1823 * 1824 * @adev: amdgpu_device pointer 1825 * @fence: fence for the callback 1826 */ 1827 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev, 1828 struct dma_fence *fence) 1829 { 1830 struct amdgpu_prt_cb *cb; 1831 1832 if (!adev->gmc.gmc_funcs->set_prt) 1833 return; 1834 1835 cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL); 1836 if (!cb) { 1837 /* Last resort when we are OOM */ 1838 if (fence) 1839 dma_fence_wait(fence, false); 1840 1841 amdgpu_vm_prt_put(adev); 1842 } else { 1843 cb->adev = adev; 1844 if (!fence || dma_fence_add_callback(fence, &cb->cb, 1845 amdgpu_vm_prt_cb)) 1846 amdgpu_vm_prt_cb(fence, &cb->cb); 1847 } 1848 } 1849 1850 /** 1851 * amdgpu_vm_free_mapping - free a mapping 1852 * 1853 * @adev: amdgpu_device pointer 1854 * @vm: requested vm 1855 * @mapping: mapping to be freed 1856 * @fence: fence of the unmap operation 1857 * 1858 * Free a mapping and make sure we decrease the PRT usage count if applicable. 1859 */ 1860 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev, 1861 struct amdgpu_vm *vm, 1862 struct amdgpu_bo_va_mapping *mapping, 1863 struct dma_fence *fence) 1864 { 1865 if (mapping->flags & AMDGPU_PTE_PRT) 1866 amdgpu_vm_add_prt_cb(adev, fence); 1867 kfree(mapping); 1868 } 1869 1870 /** 1871 * amdgpu_vm_prt_fini - finish all prt mappings 1872 * 1873 * @adev: amdgpu_device pointer 1874 * @vm: requested vm 1875 * 1876 * Register a cleanup callback to disable PRT support after VM dies. 1877 */ 1878 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1879 { 1880 struct dma_resv *resv = vm->root.base.bo->tbo.base.resv; 1881 struct dma_fence *excl, **shared; 1882 unsigned i, shared_count; 1883 int r; 1884 1885 r = dma_resv_get_fences_rcu(resv, &excl, 1886 &shared_count, &shared); 1887 if (r) { 1888 /* Not enough memory to grab the fence list, as last resort 1889 * block for all the fences to complete. 1890 */ 1891 dma_resv_wait_timeout_rcu(resv, true, false, 1892 MAX_SCHEDULE_TIMEOUT); 1893 return; 1894 } 1895 1896 /* Add a callback for each fence in the reservation object */ 1897 amdgpu_vm_prt_get(adev); 1898 amdgpu_vm_add_prt_cb(adev, excl); 1899 1900 for (i = 0; i < shared_count; ++i) { 1901 amdgpu_vm_prt_get(adev); 1902 amdgpu_vm_add_prt_cb(adev, shared[i]); 1903 } 1904 1905 kfree(shared); 1906 } 1907 1908 /** 1909 * amdgpu_vm_clear_freed - clear freed BOs in the PT 1910 * 1911 * @adev: amdgpu_device pointer 1912 * @vm: requested vm 1913 * @fence: optional resulting fence (unchanged if no work needed to be done 1914 * or if an error occurred) 1915 * 1916 * Make sure all freed BOs are cleared in the PT. 1917 * PTs have to be reserved and mutex must be locked! 1918 * 1919 * Returns: 1920 * 0 for success. 1921 * 1922 */ 1923 int amdgpu_vm_clear_freed(struct amdgpu_device *adev, 1924 struct amdgpu_vm *vm, 1925 struct dma_fence **fence) 1926 { 1927 struct amdgpu_bo_va_mapping *mapping; 1928 uint64_t init_pte_value = 0; 1929 struct dma_fence *f = NULL; 1930 int r; 1931 1932 while (!list_empty(&vm->freed)) { 1933 mapping = list_first_entry(&vm->freed, 1934 struct amdgpu_bo_va_mapping, list); 1935 list_del(&mapping->list); 1936 1937 if (vm->pte_support_ats && 1938 mapping->start < AMDGPU_GMC_HOLE_START) 1939 init_pte_value = AMDGPU_PTE_DEFAULT_ATC; 1940 1941 r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm, 1942 mapping->start, mapping->last, 1943 init_pte_value, 0, &f); 1944 amdgpu_vm_free_mapping(adev, vm, mapping, f); 1945 if (r) { 1946 dma_fence_put(f); 1947 return r; 1948 } 1949 } 1950 1951 if (fence && f) { 1952 dma_fence_put(*fence); 1953 *fence = f; 1954 } else { 1955 dma_fence_put(f); 1956 } 1957 1958 return 0; 1959 1960 } 1961 1962 /** 1963 * amdgpu_vm_handle_moved - handle moved BOs in the PT 1964 * 1965 * @adev: amdgpu_device pointer 1966 * @vm: requested vm 1967 * 1968 * Make sure all BOs which are moved are updated in the PTs. 1969 * 1970 * Returns: 1971 * 0 for success. 1972 * 1973 * PTs have to be reserved! 1974 */ 1975 int amdgpu_vm_handle_moved(struct amdgpu_device *adev, 1976 struct amdgpu_vm *vm) 1977 { 1978 struct amdgpu_bo_va *bo_va, *tmp; 1979 struct dma_resv *resv; 1980 bool clear; 1981 int r; 1982 1983 list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { 1984 /* Per VM BOs never need to bo cleared in the page tables */ 1985 r = amdgpu_vm_bo_update(adev, bo_va, false); 1986 if (r) 1987 return r; 1988 } 1989 1990 spin_lock(&vm->invalidated_lock); 1991 while (!list_empty(&vm->invalidated)) { 1992 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, 1993 base.vm_status); 1994 resv = bo_va->base.bo->tbo.base.resv; 1995 spin_unlock(&vm->invalidated_lock); 1996 1997 /* Try to reserve the BO to avoid clearing its ptes */ 1998 if (!amdgpu_vm_debug && dma_resv_trylock(resv)) 1999 clear = false; 2000 /* Somebody else is using the BO right now */ 2001 else 2002 clear = true; 2003 2004 r = amdgpu_vm_bo_update(adev, bo_va, clear); 2005 if (r) 2006 return r; 2007 2008 if (!clear) 2009 dma_resv_unlock(resv); 2010 spin_lock(&vm->invalidated_lock); 2011 } 2012 spin_unlock(&vm->invalidated_lock); 2013 2014 return 0; 2015 } 2016 2017 /** 2018 * amdgpu_vm_bo_add - add a bo to a specific vm 2019 * 2020 * @adev: amdgpu_device pointer 2021 * @vm: requested vm 2022 * @bo: amdgpu buffer object 2023 * 2024 * Add @bo into the requested vm. 2025 * Add @bo to the list of bos associated with the vm 2026 * 2027 * Returns: 2028 * Newly added bo_va or NULL for failure 2029 * 2030 * Object has to be reserved! 2031 */ 2032 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, 2033 struct amdgpu_vm *vm, 2034 struct amdgpu_bo *bo) 2035 { 2036 struct amdgpu_bo_va *bo_va; 2037 2038 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL); 2039 if (bo_va == NULL) { 2040 return NULL; 2041 } 2042 amdgpu_vm_bo_base_init(&bo_va->base, vm, bo); 2043 2044 bo_va->ref_count = 1; 2045 INIT_LIST_HEAD(&bo_va->valids); 2046 INIT_LIST_HEAD(&bo_va->invalids); 2047 2048 if (bo && amdgpu_xgmi_same_hive(adev, amdgpu_ttm_adev(bo->tbo.bdev)) && 2049 (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) { 2050 bo_va->is_xgmi = true; 2051 mutex_lock(&adev->vm_manager.lock_pstate); 2052 /* Power up XGMI if it can be potentially used */ 2053 if (++adev->vm_manager.xgmi_map_counter == 1) 2054 amdgpu_xgmi_set_pstate(adev, 1); 2055 mutex_unlock(&adev->vm_manager.lock_pstate); 2056 } 2057 2058 return bo_va; 2059 } 2060 2061 2062 /** 2063 * amdgpu_vm_bo_insert_mapping - insert a new mapping 2064 * 2065 * @adev: amdgpu_device pointer 2066 * @bo_va: bo_va to store the address 2067 * @mapping: the mapping to insert 2068 * 2069 * Insert a new mapping into all structures. 2070 */ 2071 static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev, 2072 struct amdgpu_bo_va *bo_va, 2073 struct amdgpu_bo_va_mapping *mapping) 2074 { 2075 struct amdgpu_vm *vm = bo_va->base.vm; 2076 struct amdgpu_bo *bo = bo_va->base.bo; 2077 2078 mapping->bo_va = bo_va; 2079 list_add(&mapping->list, &bo_va->invalids); 2080 amdgpu_vm_it_insert(mapping, &vm->va); 2081 2082 if (mapping->flags & AMDGPU_PTE_PRT) 2083 amdgpu_vm_prt_get(adev); 2084 2085 if (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv && 2086 !bo_va->base.moved) { 2087 list_move(&bo_va->base.vm_status, &vm->moved); 2088 } 2089 trace_amdgpu_vm_bo_map(bo_va, mapping); 2090 } 2091 2092 /** 2093 * amdgpu_vm_bo_map - map bo inside a vm 2094 * 2095 * @adev: amdgpu_device pointer 2096 * @bo_va: bo_va to store the address 2097 * @saddr: where to map the BO 2098 * @offset: requested offset in the BO 2099 * @size: BO size in bytes 2100 * @flags: attributes of pages (read/write/valid/etc.) 2101 * 2102 * Add a mapping of the BO at the specefied addr into the VM. 2103 * 2104 * Returns: 2105 * 0 for success, error for failure. 2106 * 2107 * Object has to be reserved and unreserved outside! 2108 */ 2109 int amdgpu_vm_bo_map(struct amdgpu_device *adev, 2110 struct amdgpu_bo_va *bo_va, 2111 uint64_t saddr, uint64_t offset, 2112 uint64_t size, uint64_t flags) 2113 { 2114 struct amdgpu_bo_va_mapping *mapping, *tmp; 2115 struct amdgpu_bo *bo = bo_va->base.bo; 2116 struct amdgpu_vm *vm = bo_va->base.vm; 2117 uint64_t eaddr; 2118 2119 /* validate the parameters */ 2120 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || 2121 size == 0 || size & AMDGPU_GPU_PAGE_MASK) 2122 return -EINVAL; 2123 2124 /* make sure object fit at this offset */ 2125 eaddr = saddr + size - 1; 2126 if (saddr >= eaddr || 2127 (bo && offset + size > amdgpu_bo_size(bo))) 2128 return -EINVAL; 2129 2130 saddr /= AMDGPU_GPU_PAGE_SIZE; 2131 eaddr /= AMDGPU_GPU_PAGE_SIZE; 2132 2133 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); 2134 if (tmp) { 2135 /* bo and tmp overlap, invalid addr */ 2136 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with " 2137 "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr, 2138 tmp->start, tmp->last + 1); 2139 return -EINVAL; 2140 } 2141 2142 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 2143 if (!mapping) 2144 return -ENOMEM; 2145 2146 mapping->start = saddr; 2147 mapping->last = eaddr; 2148 mapping->offset = offset; 2149 mapping->flags = flags; 2150 2151 amdgpu_vm_bo_insert_map(adev, bo_va, mapping); 2152 2153 return 0; 2154 } 2155 2156 /** 2157 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings 2158 * 2159 * @adev: amdgpu_device pointer 2160 * @bo_va: bo_va to store the address 2161 * @saddr: where to map the BO 2162 * @offset: requested offset in the BO 2163 * @size: BO size in bytes 2164 * @flags: attributes of pages (read/write/valid/etc.) 2165 * 2166 * Add a mapping of the BO at the specefied addr into the VM. Replace existing 2167 * mappings as we do so. 2168 * 2169 * Returns: 2170 * 0 for success, error for failure. 2171 * 2172 * Object has to be reserved and unreserved outside! 2173 */ 2174 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev, 2175 struct amdgpu_bo_va *bo_va, 2176 uint64_t saddr, uint64_t offset, 2177 uint64_t size, uint64_t flags) 2178 { 2179 struct amdgpu_bo_va_mapping *mapping; 2180 struct amdgpu_bo *bo = bo_va->base.bo; 2181 uint64_t eaddr; 2182 int r; 2183 2184 /* validate the parameters */ 2185 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || 2186 size == 0 || size & AMDGPU_GPU_PAGE_MASK) 2187 return -EINVAL; 2188 2189 /* make sure object fit at this offset */ 2190 eaddr = saddr + size - 1; 2191 if (saddr >= eaddr || 2192 (bo && offset + size > amdgpu_bo_size(bo))) 2193 return -EINVAL; 2194 2195 /* Allocate all the needed memory */ 2196 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 2197 if (!mapping) 2198 return -ENOMEM; 2199 2200 r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size); 2201 if (r) { 2202 kfree(mapping); 2203 return r; 2204 } 2205 2206 saddr /= AMDGPU_GPU_PAGE_SIZE; 2207 eaddr /= AMDGPU_GPU_PAGE_SIZE; 2208 2209 mapping->start = saddr; 2210 mapping->last = eaddr; 2211 mapping->offset = offset; 2212 mapping->flags = flags; 2213 2214 amdgpu_vm_bo_insert_map(adev, bo_va, mapping); 2215 2216 return 0; 2217 } 2218 2219 /** 2220 * amdgpu_vm_bo_unmap - remove bo mapping from vm 2221 * 2222 * @adev: amdgpu_device pointer 2223 * @bo_va: bo_va to remove the address from 2224 * @saddr: where to the BO is mapped 2225 * 2226 * Remove a mapping of the BO at the specefied addr from the VM. 2227 * 2228 * Returns: 2229 * 0 for success, error for failure. 2230 * 2231 * Object has to be reserved and unreserved outside! 2232 */ 2233 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, 2234 struct amdgpu_bo_va *bo_va, 2235 uint64_t saddr) 2236 { 2237 struct amdgpu_bo_va_mapping *mapping; 2238 struct amdgpu_vm *vm = bo_va->base.vm; 2239 bool valid = true; 2240 2241 saddr /= AMDGPU_GPU_PAGE_SIZE; 2242 2243 list_for_each_entry(mapping, &bo_va->valids, list) { 2244 if (mapping->start == saddr) 2245 break; 2246 } 2247 2248 if (&mapping->list == &bo_va->valids) { 2249 valid = false; 2250 2251 list_for_each_entry(mapping, &bo_va->invalids, list) { 2252 if (mapping->start == saddr) 2253 break; 2254 } 2255 2256 if (&mapping->list == &bo_va->invalids) 2257 return -ENOENT; 2258 } 2259 2260 list_del(&mapping->list); 2261 amdgpu_vm_it_remove(mapping, &vm->va); 2262 mapping->bo_va = NULL; 2263 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 2264 2265 if (valid) 2266 list_add(&mapping->list, &vm->freed); 2267 else 2268 amdgpu_vm_free_mapping(adev, vm, mapping, 2269 bo_va->last_pt_update); 2270 2271 return 0; 2272 } 2273 2274 /** 2275 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range 2276 * 2277 * @adev: amdgpu_device pointer 2278 * @vm: VM structure to use 2279 * @saddr: start of the range 2280 * @size: size of the range 2281 * 2282 * Remove all mappings in a range, split them as appropriate. 2283 * 2284 * Returns: 2285 * 0 for success, error for failure. 2286 */ 2287 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, 2288 struct amdgpu_vm *vm, 2289 uint64_t saddr, uint64_t size) 2290 { 2291 struct amdgpu_bo_va_mapping *before, *after, *tmp, *next; 2292 LIST_HEAD(removed); 2293 uint64_t eaddr; 2294 2295 eaddr = saddr + size - 1; 2296 saddr /= AMDGPU_GPU_PAGE_SIZE; 2297 eaddr /= AMDGPU_GPU_PAGE_SIZE; 2298 2299 /* Allocate all the needed memory */ 2300 before = kzalloc(sizeof(*before), GFP_KERNEL); 2301 if (!before) 2302 return -ENOMEM; 2303 INIT_LIST_HEAD(&before->list); 2304 2305 after = kzalloc(sizeof(*after), GFP_KERNEL); 2306 if (!after) { 2307 kfree(before); 2308 return -ENOMEM; 2309 } 2310 INIT_LIST_HEAD(&after->list); 2311 2312 /* Now gather all removed mappings */ 2313 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); 2314 while (tmp) { 2315 /* Remember mapping split at the start */ 2316 if (tmp->start < saddr) { 2317 before->start = tmp->start; 2318 before->last = saddr - 1; 2319 before->offset = tmp->offset; 2320 before->flags = tmp->flags; 2321 before->bo_va = tmp->bo_va; 2322 list_add(&before->list, &tmp->bo_va->invalids); 2323 } 2324 2325 /* Remember mapping split at the end */ 2326 if (tmp->last > eaddr) { 2327 after->start = eaddr + 1; 2328 after->last = tmp->last; 2329 after->offset = tmp->offset; 2330 after->offset += after->start - tmp->start; 2331 after->flags = tmp->flags; 2332 after->bo_va = tmp->bo_va; 2333 list_add(&after->list, &tmp->bo_va->invalids); 2334 } 2335 2336 list_del(&tmp->list); 2337 list_add(&tmp->list, &removed); 2338 2339 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr); 2340 } 2341 2342 /* And free them up */ 2343 list_for_each_entry_safe(tmp, next, &removed, list) { 2344 amdgpu_vm_it_remove(tmp, &vm->va); 2345 list_del(&tmp->list); 2346 2347 if (tmp->start < saddr) 2348 tmp->start = saddr; 2349 if (tmp->last > eaddr) 2350 tmp->last = eaddr; 2351 2352 tmp->bo_va = NULL; 2353 list_add(&tmp->list, &vm->freed); 2354 trace_amdgpu_vm_bo_unmap(NULL, tmp); 2355 } 2356 2357 /* Insert partial mapping before the range */ 2358 if (!list_empty(&before->list)) { 2359 amdgpu_vm_it_insert(before, &vm->va); 2360 if (before->flags & AMDGPU_PTE_PRT) 2361 amdgpu_vm_prt_get(adev); 2362 } else { 2363 kfree(before); 2364 } 2365 2366 /* Insert partial mapping after the range */ 2367 if (!list_empty(&after->list)) { 2368 amdgpu_vm_it_insert(after, &vm->va); 2369 if (after->flags & AMDGPU_PTE_PRT) 2370 amdgpu_vm_prt_get(adev); 2371 } else { 2372 kfree(after); 2373 } 2374 2375 return 0; 2376 } 2377 2378 /** 2379 * amdgpu_vm_bo_lookup_mapping - find mapping by address 2380 * 2381 * @vm: the requested VM 2382 * @addr: the address 2383 * 2384 * Find a mapping by it's address. 2385 * 2386 * Returns: 2387 * The amdgpu_bo_va_mapping matching for addr or NULL 2388 * 2389 */ 2390 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm, 2391 uint64_t addr) 2392 { 2393 return amdgpu_vm_it_iter_first(&vm->va, addr, addr); 2394 } 2395 2396 /** 2397 * amdgpu_vm_bo_trace_cs - trace all reserved mappings 2398 * 2399 * @vm: the requested vm 2400 * @ticket: CS ticket 2401 * 2402 * Trace all mappings of BOs reserved during a command submission. 2403 */ 2404 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket) 2405 { 2406 struct amdgpu_bo_va_mapping *mapping; 2407 2408 if (!trace_amdgpu_vm_bo_cs_enabled()) 2409 return; 2410 2411 for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping; 2412 mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) { 2413 if (mapping->bo_va && mapping->bo_va->base.bo) { 2414 struct amdgpu_bo *bo; 2415 2416 bo = mapping->bo_va->base.bo; 2417 if (dma_resv_locking_ctx(bo->tbo.base.resv) != 2418 ticket) 2419 continue; 2420 } 2421 2422 trace_amdgpu_vm_bo_cs(mapping); 2423 } 2424 } 2425 2426 /** 2427 * amdgpu_vm_bo_rmv - remove a bo to a specific vm 2428 * 2429 * @adev: amdgpu_device pointer 2430 * @bo_va: requested bo_va 2431 * 2432 * Remove @bo_va->bo from the requested vm. 2433 * 2434 * Object have to be reserved! 2435 */ 2436 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, 2437 struct amdgpu_bo_va *bo_va) 2438 { 2439 struct amdgpu_bo_va_mapping *mapping, *next; 2440 struct amdgpu_bo *bo = bo_va->base.bo; 2441 struct amdgpu_vm *vm = bo_va->base.vm; 2442 struct amdgpu_vm_bo_base **base; 2443 2444 if (bo) { 2445 if (bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) 2446 vm->bulk_moveable = false; 2447 2448 for (base = &bo_va->base.bo->vm_bo; *base; 2449 base = &(*base)->next) { 2450 if (*base != &bo_va->base) 2451 continue; 2452 2453 *base = bo_va->base.next; 2454 break; 2455 } 2456 } 2457 2458 spin_lock(&vm->invalidated_lock); 2459 list_del(&bo_va->base.vm_status); 2460 spin_unlock(&vm->invalidated_lock); 2461 2462 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 2463 list_del(&mapping->list); 2464 amdgpu_vm_it_remove(mapping, &vm->va); 2465 mapping->bo_va = NULL; 2466 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 2467 list_add(&mapping->list, &vm->freed); 2468 } 2469 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { 2470 list_del(&mapping->list); 2471 amdgpu_vm_it_remove(mapping, &vm->va); 2472 amdgpu_vm_free_mapping(adev, vm, mapping, 2473 bo_va->last_pt_update); 2474 } 2475 2476 dma_fence_put(bo_va->last_pt_update); 2477 2478 if (bo && bo_va->is_xgmi) { 2479 mutex_lock(&adev->vm_manager.lock_pstate); 2480 if (--adev->vm_manager.xgmi_map_counter == 0) 2481 amdgpu_xgmi_set_pstate(adev, 0); 2482 mutex_unlock(&adev->vm_manager.lock_pstate); 2483 } 2484 2485 kfree(bo_va); 2486 } 2487 2488 /** 2489 * amdgpu_vm_bo_invalidate - mark the bo as invalid 2490 * 2491 * @adev: amdgpu_device pointer 2492 * @bo: amdgpu buffer object 2493 * @evicted: is the BO evicted 2494 * 2495 * Mark @bo as invalid. 2496 */ 2497 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, 2498 struct amdgpu_bo *bo, bool evicted) 2499 { 2500 struct amdgpu_vm_bo_base *bo_base; 2501 2502 /* shadow bo doesn't have bo base, its validation needs its parent */ 2503 if (bo->parent && bo->parent->shadow == bo) 2504 bo = bo->parent; 2505 2506 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { 2507 struct amdgpu_vm *vm = bo_base->vm; 2508 2509 if (evicted && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) { 2510 amdgpu_vm_bo_evicted(bo_base); 2511 continue; 2512 } 2513 2514 if (bo_base->moved) 2515 continue; 2516 bo_base->moved = true; 2517 2518 if (bo->tbo.type == ttm_bo_type_kernel) 2519 amdgpu_vm_bo_relocated(bo_base); 2520 else if (bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) 2521 amdgpu_vm_bo_moved(bo_base); 2522 else 2523 amdgpu_vm_bo_invalidated(bo_base); 2524 } 2525 } 2526 2527 /** 2528 * amdgpu_vm_get_block_size - calculate VM page table size as power of two 2529 * 2530 * @vm_size: VM size 2531 * 2532 * Returns: 2533 * VM page table as power of two 2534 */ 2535 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size) 2536 { 2537 /* Total bits covered by PD + PTs */ 2538 unsigned bits = ilog2(vm_size) + 18; 2539 2540 /* Make sure the PD is 4K in size up to 8GB address space. 2541 Above that split equal between PD and PTs */ 2542 if (vm_size <= 8) 2543 return (bits - 9); 2544 else 2545 return ((bits + 3) / 2); 2546 } 2547 2548 /** 2549 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size 2550 * 2551 * @adev: amdgpu_device pointer 2552 * @min_vm_size: the minimum vm size in GB if it's set auto 2553 * @fragment_size_default: Default PTE fragment size 2554 * @max_level: max VMPT level 2555 * @max_bits: max address space size in bits 2556 * 2557 */ 2558 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size, 2559 uint32_t fragment_size_default, unsigned max_level, 2560 unsigned max_bits) 2561 { 2562 unsigned int max_size = 1 << (max_bits - 30); 2563 unsigned int vm_size; 2564 uint64_t tmp; 2565 2566 /* adjust vm size first */ 2567 if (amdgpu_vm_size != -1) { 2568 vm_size = amdgpu_vm_size; 2569 if (vm_size > max_size) { 2570 dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n", 2571 amdgpu_vm_size, max_size); 2572 vm_size = max_size; 2573 } 2574 } else { 2575 struct sysinfo si; 2576 unsigned int phys_ram_gb; 2577 2578 /* Optimal VM size depends on the amount of physical 2579 * RAM available. Underlying requirements and 2580 * assumptions: 2581 * 2582 * - Need to map system memory and VRAM from all GPUs 2583 * - VRAM from other GPUs not known here 2584 * - Assume VRAM <= system memory 2585 * - On GFX8 and older, VM space can be segmented for 2586 * different MTYPEs 2587 * - Need to allow room for fragmentation, guard pages etc. 2588 * 2589 * This adds up to a rough guess of system memory x3. 2590 * Round up to power of two to maximize the available 2591 * VM size with the given page table size. 2592 */ 2593 si_meminfo(&si); 2594 phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit + 2595 (1 << 30) - 1) >> 30; 2596 vm_size = roundup_pow_of_two( 2597 min(max(phys_ram_gb * 3, min_vm_size), max_size)); 2598 } 2599 2600 adev->vm_manager.max_pfn = (uint64_t)vm_size << 18; 2601 2602 tmp = roundup_pow_of_two(adev->vm_manager.max_pfn); 2603 if (amdgpu_vm_block_size != -1) 2604 tmp >>= amdgpu_vm_block_size - 9; 2605 tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1; 2606 adev->vm_manager.num_level = min(max_level, (unsigned)tmp); 2607 switch (adev->vm_manager.num_level) { 2608 case 3: 2609 adev->vm_manager.root_level = AMDGPU_VM_PDB2; 2610 break; 2611 case 2: 2612 adev->vm_manager.root_level = AMDGPU_VM_PDB1; 2613 break; 2614 case 1: 2615 adev->vm_manager.root_level = AMDGPU_VM_PDB0; 2616 break; 2617 default: 2618 dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n"); 2619 } 2620 /* block size depends on vm size and hw setup*/ 2621 if (amdgpu_vm_block_size != -1) 2622 adev->vm_manager.block_size = 2623 min((unsigned)amdgpu_vm_block_size, max_bits 2624 - AMDGPU_GPU_PAGE_SHIFT 2625 - 9 * adev->vm_manager.num_level); 2626 else if (adev->vm_manager.num_level > 1) 2627 adev->vm_manager.block_size = 9; 2628 else 2629 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp); 2630 2631 if (amdgpu_vm_fragment_size == -1) 2632 adev->vm_manager.fragment_size = fragment_size_default; 2633 else 2634 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size; 2635 2636 DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n", 2637 vm_size, adev->vm_manager.num_level + 1, 2638 adev->vm_manager.block_size, 2639 adev->vm_manager.fragment_size); 2640 } 2641 2642 /** 2643 * amdgpu_vm_wait_idle - wait for the VM to become idle 2644 * 2645 * @vm: VM object to wait for 2646 * @timeout: timeout to wait for VM to become idle 2647 */ 2648 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout) 2649 { 2650 return dma_resv_wait_timeout_rcu(vm->root.base.bo->tbo.base.resv, 2651 true, true, timeout); 2652 } 2653 2654 /** 2655 * amdgpu_vm_init - initialize a vm instance 2656 * 2657 * @adev: amdgpu_device pointer 2658 * @vm: requested vm 2659 * @vm_context: Indicates if it GFX or Compute context 2660 * @pasid: Process address space identifier 2661 * 2662 * Init @vm fields. 2663 * 2664 * Returns: 2665 * 0 for success, error for failure. 2666 */ 2667 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, 2668 int vm_context, unsigned int pasid) 2669 { 2670 struct amdgpu_bo_param bp; 2671 struct amdgpu_bo *root; 2672 int r, i; 2673 2674 vm->va = RB_ROOT_CACHED; 2675 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) 2676 vm->reserved_vmid[i] = NULL; 2677 INIT_LIST_HEAD(&vm->evicted); 2678 INIT_LIST_HEAD(&vm->relocated); 2679 INIT_LIST_HEAD(&vm->moved); 2680 INIT_LIST_HEAD(&vm->idle); 2681 INIT_LIST_HEAD(&vm->invalidated); 2682 spin_lock_init(&vm->invalidated_lock); 2683 INIT_LIST_HEAD(&vm->freed); 2684 2685 /* create scheduler entity for page table updates */ 2686 r = drm_sched_entity_init(&vm->entity, adev->vm_manager.vm_pte_rqs, 2687 adev->vm_manager.vm_pte_num_rqs, NULL); 2688 if (r) 2689 return r; 2690 2691 vm->pte_support_ats = false; 2692 2693 if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) { 2694 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2695 AMDGPU_VM_USE_CPU_FOR_COMPUTE); 2696 2697 if (adev->asic_type == CHIP_RAVEN) 2698 vm->pte_support_ats = true; 2699 } else { 2700 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2701 AMDGPU_VM_USE_CPU_FOR_GFX); 2702 } 2703 DRM_DEBUG_DRIVER("VM update mode is %s\n", 2704 vm->use_cpu_for_update ? "CPU" : "SDMA"); 2705 WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)), 2706 "CPU update of VM recommended only for large BAR system\n"); 2707 2708 if (vm->use_cpu_for_update) 2709 vm->update_funcs = &amdgpu_vm_cpu_funcs; 2710 else 2711 vm->update_funcs = &amdgpu_vm_sdma_funcs; 2712 vm->last_update = NULL; 2713 2714 amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, &bp); 2715 if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) 2716 bp.flags &= ~AMDGPU_GEM_CREATE_SHADOW; 2717 r = amdgpu_bo_create(adev, &bp, &root); 2718 if (r) 2719 goto error_free_sched_entity; 2720 2721 r = amdgpu_bo_reserve(root, true); 2722 if (r) 2723 goto error_free_root; 2724 2725 r = dma_resv_reserve_shared(root->tbo.base.resv, 1); 2726 if (r) 2727 goto error_unreserve; 2728 2729 amdgpu_vm_bo_base_init(&vm->root.base, vm, root); 2730 2731 r = amdgpu_vm_clear_bo(adev, vm, root); 2732 if (r) 2733 goto error_unreserve; 2734 2735 amdgpu_bo_unreserve(vm->root.base.bo); 2736 2737 if (pasid) { 2738 unsigned long flags; 2739 2740 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2741 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1, 2742 GFP_ATOMIC); 2743 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2744 if (r < 0) 2745 goto error_free_root; 2746 2747 vm->pasid = pasid; 2748 } 2749 2750 INIT_KFIFO(vm->faults); 2751 2752 return 0; 2753 2754 error_unreserve: 2755 amdgpu_bo_unreserve(vm->root.base.bo); 2756 2757 error_free_root: 2758 amdgpu_bo_unref(&vm->root.base.bo->shadow); 2759 amdgpu_bo_unref(&vm->root.base.bo); 2760 vm->root.base.bo = NULL; 2761 2762 error_free_sched_entity: 2763 drm_sched_entity_destroy(&vm->entity); 2764 2765 return r; 2766 } 2767 2768 /** 2769 * amdgpu_vm_check_clean_reserved - check if a VM is clean 2770 * 2771 * @adev: amdgpu_device pointer 2772 * @vm: the VM to check 2773 * 2774 * check all entries of the root PD, if any subsequent PDs are allocated, 2775 * it means there are page table creating and filling, and is no a clean 2776 * VM 2777 * 2778 * Returns: 2779 * 0 if this VM is clean 2780 */ 2781 static int amdgpu_vm_check_clean_reserved(struct amdgpu_device *adev, 2782 struct amdgpu_vm *vm) 2783 { 2784 enum amdgpu_vm_level root = adev->vm_manager.root_level; 2785 unsigned int entries = amdgpu_vm_num_entries(adev, root); 2786 unsigned int i = 0; 2787 2788 if (!(vm->root.entries)) 2789 return 0; 2790 2791 for (i = 0; i < entries; i++) { 2792 if (vm->root.entries[i].base.bo) 2793 return -EINVAL; 2794 } 2795 2796 return 0; 2797 } 2798 2799 /** 2800 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM 2801 * 2802 * @adev: amdgpu_device pointer 2803 * @vm: requested vm 2804 * 2805 * This only works on GFX VMs that don't have any BOs added and no 2806 * page tables allocated yet. 2807 * 2808 * Changes the following VM parameters: 2809 * - use_cpu_for_update 2810 * - pte_supports_ats 2811 * - pasid (old PASID is released, because compute manages its own PASIDs) 2812 * 2813 * Reinitializes the page directory to reflect the changed ATS 2814 * setting. 2815 * 2816 * Returns: 2817 * 0 for success, -errno for errors. 2818 */ 2819 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned int pasid) 2820 { 2821 bool pte_support_ats = (adev->asic_type == CHIP_RAVEN); 2822 int r; 2823 2824 r = amdgpu_bo_reserve(vm->root.base.bo, true); 2825 if (r) 2826 return r; 2827 2828 /* Sanity checks */ 2829 r = amdgpu_vm_check_clean_reserved(adev, vm); 2830 if (r) 2831 goto unreserve_bo; 2832 2833 if (pasid) { 2834 unsigned long flags; 2835 2836 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2837 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1, 2838 GFP_ATOMIC); 2839 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2840 2841 if (r == -ENOSPC) 2842 goto unreserve_bo; 2843 r = 0; 2844 } 2845 2846 /* Check if PD needs to be reinitialized and do it before 2847 * changing any other state, in case it fails. 2848 */ 2849 if (pte_support_ats != vm->pte_support_ats) { 2850 vm->pte_support_ats = pte_support_ats; 2851 r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo); 2852 if (r) 2853 goto free_idr; 2854 } 2855 2856 /* Update VM state */ 2857 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2858 AMDGPU_VM_USE_CPU_FOR_COMPUTE); 2859 DRM_DEBUG_DRIVER("VM update mode is %s\n", 2860 vm->use_cpu_for_update ? "CPU" : "SDMA"); 2861 WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)), 2862 "CPU update of VM recommended only for large BAR system\n"); 2863 2864 if (vm->use_cpu_for_update) 2865 vm->update_funcs = &amdgpu_vm_cpu_funcs; 2866 else 2867 vm->update_funcs = &amdgpu_vm_sdma_funcs; 2868 dma_fence_put(vm->last_update); 2869 vm->last_update = NULL; 2870 2871 if (vm->pasid) { 2872 unsigned long flags; 2873 2874 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2875 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); 2876 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2877 2878 /* Free the original amdgpu allocated pasid 2879 * Will be replaced with kfd allocated pasid 2880 */ 2881 amdgpu_pasid_free(vm->pasid); 2882 vm->pasid = 0; 2883 } 2884 2885 /* Free the shadow bo for compute VM */ 2886 amdgpu_bo_unref(&vm->root.base.bo->shadow); 2887 2888 if (pasid) 2889 vm->pasid = pasid; 2890 2891 goto unreserve_bo; 2892 2893 free_idr: 2894 if (pasid) { 2895 unsigned long flags; 2896 2897 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2898 idr_remove(&adev->vm_manager.pasid_idr, pasid); 2899 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2900 } 2901 unreserve_bo: 2902 amdgpu_bo_unreserve(vm->root.base.bo); 2903 return r; 2904 } 2905 2906 /** 2907 * amdgpu_vm_release_compute - release a compute vm 2908 * @adev: amdgpu_device pointer 2909 * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute 2910 * 2911 * This is a correspondant of amdgpu_vm_make_compute. It decouples compute 2912 * pasid from vm. Compute should stop use of vm after this call. 2913 */ 2914 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) 2915 { 2916 if (vm->pasid) { 2917 unsigned long flags; 2918 2919 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2920 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); 2921 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2922 } 2923 vm->pasid = 0; 2924 } 2925 2926 /** 2927 * amdgpu_vm_fini - tear down a vm instance 2928 * 2929 * @adev: amdgpu_device pointer 2930 * @vm: requested vm 2931 * 2932 * Tear down @vm. 2933 * Unbind the VM and remove all bos from the vm bo list 2934 */ 2935 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 2936 { 2937 struct amdgpu_bo_va_mapping *mapping, *tmp; 2938 bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt; 2939 struct amdgpu_bo *root; 2940 int i, r; 2941 2942 amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm); 2943 2944 if (vm->pasid) { 2945 unsigned long flags; 2946 2947 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2948 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); 2949 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2950 } 2951 2952 drm_sched_entity_destroy(&vm->entity); 2953 2954 if (!RB_EMPTY_ROOT(&vm->va.rb_root)) { 2955 dev_err(adev->dev, "still active bo inside vm\n"); 2956 } 2957 rbtree_postorder_for_each_entry_safe(mapping, tmp, 2958 &vm->va.rb_root, rb) { 2959 /* Don't remove the mapping here, we don't want to trigger a 2960 * rebalance and the tree is about to be destroyed anyway. 2961 */ 2962 list_del(&mapping->list); 2963 kfree(mapping); 2964 } 2965 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { 2966 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) { 2967 amdgpu_vm_prt_fini(adev, vm); 2968 prt_fini_needed = false; 2969 } 2970 2971 list_del(&mapping->list); 2972 amdgpu_vm_free_mapping(adev, vm, mapping, NULL); 2973 } 2974 2975 root = amdgpu_bo_ref(vm->root.base.bo); 2976 r = amdgpu_bo_reserve(root, true); 2977 if (r) { 2978 dev_err(adev->dev, "Leaking page tables because BO reservation failed\n"); 2979 } else { 2980 amdgpu_vm_free_pts(adev, vm, NULL); 2981 amdgpu_bo_unreserve(root); 2982 } 2983 amdgpu_bo_unref(&root); 2984 WARN_ON(vm->root.base.bo); 2985 dma_fence_put(vm->last_update); 2986 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) 2987 amdgpu_vmid_free_reserved(adev, vm, i); 2988 } 2989 2990 /** 2991 * amdgpu_vm_manager_init - init the VM manager 2992 * 2993 * @adev: amdgpu_device pointer 2994 * 2995 * Initialize the VM manager structures 2996 */ 2997 void amdgpu_vm_manager_init(struct amdgpu_device *adev) 2998 { 2999 unsigned i; 3000 3001 amdgpu_vmid_mgr_init(adev); 3002 3003 adev->vm_manager.fence_context = 3004 dma_fence_context_alloc(AMDGPU_MAX_RINGS); 3005 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) 3006 adev->vm_manager.seqno[i] = 0; 3007 3008 spin_lock_init(&adev->vm_manager.prt_lock); 3009 atomic_set(&adev->vm_manager.num_prt_users, 0); 3010 3011 /* If not overridden by the user, by default, only in large BAR systems 3012 * Compute VM tables will be updated by CPU 3013 */ 3014 #ifdef CONFIG_X86_64 3015 if (amdgpu_vm_update_mode == -1) { 3016 if (amdgpu_gmc_vram_full_visible(&adev->gmc)) 3017 adev->vm_manager.vm_update_mode = 3018 AMDGPU_VM_USE_CPU_FOR_COMPUTE; 3019 else 3020 adev->vm_manager.vm_update_mode = 0; 3021 } else 3022 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode; 3023 #else 3024 adev->vm_manager.vm_update_mode = 0; 3025 #endif 3026 3027 idr_init(&adev->vm_manager.pasid_idr); 3028 spin_lock_init(&adev->vm_manager.pasid_lock); 3029 3030 adev->vm_manager.xgmi_map_counter = 0; 3031 mutex_init(&adev->vm_manager.lock_pstate); 3032 } 3033 3034 /** 3035 * amdgpu_vm_manager_fini - cleanup VM manager 3036 * 3037 * @adev: amdgpu_device pointer 3038 * 3039 * Cleanup the VM manager and free resources. 3040 */ 3041 void amdgpu_vm_manager_fini(struct amdgpu_device *adev) 3042 { 3043 WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr)); 3044 idr_destroy(&adev->vm_manager.pasid_idr); 3045 3046 amdgpu_vmid_mgr_fini(adev); 3047 } 3048 3049 /** 3050 * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs. 3051 * 3052 * @dev: drm device pointer 3053 * @data: drm_amdgpu_vm 3054 * @filp: drm file pointer 3055 * 3056 * Returns: 3057 * 0 for success, -errno for errors. 3058 */ 3059 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 3060 { 3061 union drm_amdgpu_vm *args = data; 3062 struct amdgpu_device *adev = dev->dev_private; 3063 struct amdgpu_fpriv *fpriv = filp->driver_priv; 3064 int r; 3065 3066 switch (args->in.op) { 3067 case AMDGPU_VM_OP_RESERVE_VMID: 3068 /* current, we only have requirement to reserve vmid from gfxhub */ 3069 r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0); 3070 if (r) 3071 return r; 3072 break; 3073 case AMDGPU_VM_OP_UNRESERVE_VMID: 3074 amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0); 3075 break; 3076 default: 3077 return -EINVAL; 3078 } 3079 3080 return 0; 3081 } 3082 3083 /** 3084 * amdgpu_vm_get_task_info - Extracts task info for a PASID. 3085 * 3086 * @adev: drm device pointer 3087 * @pasid: PASID identifier for VM 3088 * @task_info: task_info to fill. 3089 */ 3090 void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid, 3091 struct amdgpu_task_info *task_info) 3092 { 3093 struct amdgpu_vm *vm; 3094 unsigned long flags; 3095 3096 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 3097 3098 vm = idr_find(&adev->vm_manager.pasid_idr, pasid); 3099 if (vm) 3100 *task_info = vm->task_info; 3101 3102 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 3103 } 3104 3105 /** 3106 * amdgpu_vm_set_task_info - Sets VMs task info. 3107 * 3108 * @vm: vm for which to set the info 3109 */ 3110 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm) 3111 { 3112 if (!vm->task_info.pid) { 3113 vm->task_info.pid = current->pid; 3114 get_task_comm(vm->task_info.task_name, current); 3115 3116 if (current->group_leader->mm == current->mm) { 3117 vm->task_info.tgid = current->group_leader->pid; 3118 get_task_comm(vm->task_info.process_name, current->group_leader); 3119 } 3120 } 3121 } 3122