1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <linux/dma-fence-array.h> 29 #include <linux/interval_tree_generic.h> 30 #include <linux/idr.h> 31 32 #include <drm/amdgpu_drm.h> 33 #include "amdgpu.h" 34 #include "amdgpu_trace.h" 35 #include "amdgpu_amdkfd.h" 36 #include "amdgpu_gmc.h" 37 #include "amdgpu_xgmi.h" 38 39 /** 40 * DOC: GPUVM 41 * 42 * GPUVM is similar to the legacy gart on older asics, however 43 * rather than there being a single global gart table 44 * for the entire GPU, there are multiple VM page tables active 45 * at any given time. The VM page tables can contain a mix 46 * vram pages and system memory pages and system memory pages 47 * can be mapped as snooped (cached system pages) or unsnooped 48 * (uncached system pages). 49 * Each VM has an ID associated with it and there is a page table 50 * associated with each VMID. When execting a command buffer, 51 * the kernel tells the the ring what VMID to use for that command 52 * buffer. VMIDs are allocated dynamically as commands are submitted. 53 * The userspace drivers maintain their own address space and the kernel 54 * sets up their pages tables accordingly when they submit their 55 * command buffers and a VMID is assigned. 56 * Cayman/Trinity support up to 8 active VMs at any given time; 57 * SI supports 16. 58 */ 59 60 #define START(node) ((node)->start) 61 #define LAST(node) ((node)->last) 62 63 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, 64 START, LAST, static, amdgpu_vm_it) 65 66 #undef START 67 #undef LAST 68 69 /** 70 * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback 71 */ 72 struct amdgpu_prt_cb { 73 74 /** 75 * @adev: amdgpu device 76 */ 77 struct amdgpu_device *adev; 78 79 /** 80 * @cb: callback 81 */ 82 struct dma_fence_cb cb; 83 }; 84 85 /** 86 * amdgpu_vm_level_shift - return the addr shift for each level 87 * 88 * @adev: amdgpu_device pointer 89 * @level: VMPT level 90 * 91 * Returns: 92 * The number of bits the pfn needs to be right shifted for a level. 93 */ 94 static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev, 95 unsigned level) 96 { 97 unsigned shift = 0xff; 98 99 switch (level) { 100 case AMDGPU_VM_PDB2: 101 case AMDGPU_VM_PDB1: 102 case AMDGPU_VM_PDB0: 103 shift = 9 * (AMDGPU_VM_PDB0 - level) + 104 adev->vm_manager.block_size; 105 break; 106 case AMDGPU_VM_PTB: 107 shift = 0; 108 break; 109 default: 110 dev_err(adev->dev, "the level%d isn't supported.\n", level); 111 } 112 113 return shift; 114 } 115 116 /** 117 * amdgpu_vm_num_entries - return the number of entries in a PD/PT 118 * 119 * @adev: amdgpu_device pointer 120 * @level: VMPT level 121 * 122 * Returns: 123 * The number of entries in a page directory or page table. 124 */ 125 static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev, 126 unsigned level) 127 { 128 unsigned shift = amdgpu_vm_level_shift(adev, 129 adev->vm_manager.root_level); 130 131 if (level == adev->vm_manager.root_level) 132 /* For the root directory */ 133 return round_up(adev->vm_manager.max_pfn, 1ULL << shift) >> shift; 134 else if (level != AMDGPU_VM_PTB) 135 /* Everything in between */ 136 return 512; 137 else 138 /* For the page tables on the leaves */ 139 return AMDGPU_VM_PTE_COUNT(adev); 140 } 141 142 /** 143 * amdgpu_vm_num_ats_entries - return the number of ATS entries in the root PD 144 * 145 * @adev: amdgpu_device pointer 146 * 147 * Returns: 148 * The number of entries in the root page directory which needs the ATS setting. 149 */ 150 static unsigned amdgpu_vm_num_ats_entries(struct amdgpu_device *adev) 151 { 152 unsigned shift; 153 154 shift = amdgpu_vm_level_shift(adev, adev->vm_manager.root_level); 155 return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT); 156 } 157 158 /** 159 * amdgpu_vm_entries_mask - the mask to get the entry number of a PD/PT 160 * 161 * @adev: amdgpu_device pointer 162 * @level: VMPT level 163 * 164 * Returns: 165 * The mask to extract the entry number of a PD/PT from an address. 166 */ 167 static uint32_t amdgpu_vm_entries_mask(struct amdgpu_device *adev, 168 unsigned int level) 169 { 170 if (level <= adev->vm_manager.root_level) 171 return 0xffffffff; 172 else if (level != AMDGPU_VM_PTB) 173 return 0x1ff; 174 else 175 return AMDGPU_VM_PTE_COUNT(adev) - 1; 176 } 177 178 /** 179 * amdgpu_vm_bo_size - returns the size of the BOs in bytes 180 * 181 * @adev: amdgpu_device pointer 182 * @level: VMPT level 183 * 184 * Returns: 185 * The size of the BO for a page directory or page table in bytes. 186 */ 187 static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level) 188 { 189 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8); 190 } 191 192 /** 193 * amdgpu_vm_bo_evicted - vm_bo is evicted 194 * 195 * @vm_bo: vm_bo which is evicted 196 * 197 * State for PDs/PTs and per VM BOs which are not at the location they should 198 * be. 199 */ 200 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) 201 { 202 struct amdgpu_vm *vm = vm_bo->vm; 203 struct amdgpu_bo *bo = vm_bo->bo; 204 205 vm_bo->moved = true; 206 if (bo->tbo.type == ttm_bo_type_kernel) 207 list_move(&vm_bo->vm_status, &vm->evicted); 208 else 209 list_move_tail(&vm_bo->vm_status, &vm->evicted); 210 } 211 212 /** 213 * amdgpu_vm_bo_relocated - vm_bo is reloacted 214 * 215 * @vm_bo: vm_bo which is relocated 216 * 217 * State for PDs/PTs which needs to update their parent PD. 218 */ 219 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) 220 { 221 list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); 222 } 223 224 /** 225 * amdgpu_vm_bo_moved - vm_bo is moved 226 * 227 * @vm_bo: vm_bo which is moved 228 * 229 * State for per VM BOs which are moved, but that change is not yet reflected 230 * in the page tables. 231 */ 232 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) 233 { 234 list_move(&vm_bo->vm_status, &vm_bo->vm->moved); 235 } 236 237 /** 238 * amdgpu_vm_bo_idle - vm_bo is idle 239 * 240 * @vm_bo: vm_bo which is now idle 241 * 242 * State for PDs/PTs and per VM BOs which have gone through the state machine 243 * and are now idle. 244 */ 245 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) 246 { 247 list_move(&vm_bo->vm_status, &vm_bo->vm->idle); 248 vm_bo->moved = false; 249 } 250 251 /** 252 * amdgpu_vm_bo_invalidated - vm_bo is invalidated 253 * 254 * @vm_bo: vm_bo which is now invalidated 255 * 256 * State for normal BOs which are invalidated and that change not yet reflected 257 * in the PTs. 258 */ 259 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) 260 { 261 spin_lock(&vm_bo->vm->invalidated_lock); 262 list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); 263 spin_unlock(&vm_bo->vm->invalidated_lock); 264 } 265 266 /** 267 * amdgpu_vm_bo_done - vm_bo is done 268 * 269 * @vm_bo: vm_bo which is now done 270 * 271 * State for normal BOs which are invalidated and that change has been updated 272 * in the PTs. 273 */ 274 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo) 275 { 276 spin_lock(&vm_bo->vm->invalidated_lock); 277 list_del_init(&vm_bo->vm_status); 278 spin_unlock(&vm_bo->vm->invalidated_lock); 279 } 280 281 /** 282 * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm 283 * 284 * @base: base structure for tracking BO usage in a VM 285 * @vm: vm to which bo is to be added 286 * @bo: amdgpu buffer object 287 * 288 * Initialize a bo_va_base structure and add it to the appropriate lists 289 * 290 */ 291 static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, 292 struct amdgpu_vm *vm, 293 struct amdgpu_bo *bo) 294 { 295 base->vm = vm; 296 base->bo = bo; 297 base->next = NULL; 298 INIT_LIST_HEAD(&base->vm_status); 299 300 if (!bo) 301 return; 302 base->next = bo->vm_bo; 303 bo->vm_bo = base; 304 305 if (bo->tbo.base.resv != vm->root.base.bo->tbo.base.resv) 306 return; 307 308 vm->bulk_moveable = false; 309 if (bo->tbo.type == ttm_bo_type_kernel && bo->parent) 310 amdgpu_vm_bo_relocated(base); 311 else 312 amdgpu_vm_bo_idle(base); 313 314 if (bo->preferred_domains & 315 amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type)) 316 return; 317 318 /* 319 * we checked all the prerequisites, but it looks like this per vm bo 320 * is currently evicted. add the bo to the evicted list to make sure it 321 * is validated on next vm use to avoid fault. 322 * */ 323 amdgpu_vm_bo_evicted(base); 324 } 325 326 /** 327 * amdgpu_vm_pt_parent - get the parent page directory 328 * 329 * @pt: child page table 330 * 331 * Helper to get the parent entry for the child page table. NULL if we are at 332 * the root page directory. 333 */ 334 static struct amdgpu_vm_pt *amdgpu_vm_pt_parent(struct amdgpu_vm_pt *pt) 335 { 336 struct amdgpu_bo *parent = pt->base.bo->parent; 337 338 if (!parent) 339 return NULL; 340 341 return container_of(parent->vm_bo, struct amdgpu_vm_pt, base); 342 } 343 344 /** 345 * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt 346 */ 347 struct amdgpu_vm_pt_cursor { 348 uint64_t pfn; 349 struct amdgpu_vm_pt *parent; 350 struct amdgpu_vm_pt *entry; 351 unsigned level; 352 }; 353 354 /** 355 * amdgpu_vm_pt_start - start PD/PT walk 356 * 357 * @adev: amdgpu_device pointer 358 * @vm: amdgpu_vm structure 359 * @start: start address of the walk 360 * @cursor: state to initialize 361 * 362 * Initialize a amdgpu_vm_pt_cursor to start a walk. 363 */ 364 static void amdgpu_vm_pt_start(struct amdgpu_device *adev, 365 struct amdgpu_vm *vm, uint64_t start, 366 struct amdgpu_vm_pt_cursor *cursor) 367 { 368 cursor->pfn = start; 369 cursor->parent = NULL; 370 cursor->entry = &vm->root; 371 cursor->level = adev->vm_manager.root_level; 372 } 373 374 /** 375 * amdgpu_vm_pt_descendant - go to child node 376 * 377 * @adev: amdgpu_device pointer 378 * @cursor: current state 379 * 380 * Walk to the child node of the current node. 381 * Returns: 382 * True if the walk was possible, false otherwise. 383 */ 384 static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev, 385 struct amdgpu_vm_pt_cursor *cursor) 386 { 387 unsigned mask, shift, idx; 388 389 if (!cursor->entry->entries) 390 return false; 391 392 BUG_ON(!cursor->entry->base.bo); 393 mask = amdgpu_vm_entries_mask(adev, cursor->level); 394 shift = amdgpu_vm_level_shift(adev, cursor->level); 395 396 ++cursor->level; 397 idx = (cursor->pfn >> shift) & mask; 398 cursor->parent = cursor->entry; 399 cursor->entry = &cursor->entry->entries[idx]; 400 return true; 401 } 402 403 /** 404 * amdgpu_vm_pt_sibling - go to sibling node 405 * 406 * @adev: amdgpu_device pointer 407 * @cursor: current state 408 * 409 * Walk to the sibling node of the current node. 410 * Returns: 411 * True if the walk was possible, false otherwise. 412 */ 413 static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev, 414 struct amdgpu_vm_pt_cursor *cursor) 415 { 416 unsigned shift, num_entries; 417 418 /* Root doesn't have a sibling */ 419 if (!cursor->parent) 420 return false; 421 422 /* Go to our parents and see if we got a sibling */ 423 shift = amdgpu_vm_level_shift(adev, cursor->level - 1); 424 num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1); 425 426 if (cursor->entry == &cursor->parent->entries[num_entries - 1]) 427 return false; 428 429 cursor->pfn += 1ULL << shift; 430 cursor->pfn &= ~((1ULL << shift) - 1); 431 ++cursor->entry; 432 return true; 433 } 434 435 /** 436 * amdgpu_vm_pt_ancestor - go to parent node 437 * 438 * @cursor: current state 439 * 440 * Walk to the parent node of the current node. 441 * Returns: 442 * True if the walk was possible, false otherwise. 443 */ 444 static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor) 445 { 446 if (!cursor->parent) 447 return false; 448 449 --cursor->level; 450 cursor->entry = cursor->parent; 451 cursor->parent = amdgpu_vm_pt_parent(cursor->parent); 452 return true; 453 } 454 455 /** 456 * amdgpu_vm_pt_next - get next PD/PT in hieratchy 457 * 458 * @adev: amdgpu_device pointer 459 * @cursor: current state 460 * 461 * Walk the PD/PT tree to the next node. 462 */ 463 static void amdgpu_vm_pt_next(struct amdgpu_device *adev, 464 struct amdgpu_vm_pt_cursor *cursor) 465 { 466 /* First try a newborn child */ 467 if (amdgpu_vm_pt_descendant(adev, cursor)) 468 return; 469 470 /* If that didn't worked try to find a sibling */ 471 while (!amdgpu_vm_pt_sibling(adev, cursor)) { 472 /* No sibling, go to our parents and grandparents */ 473 if (!amdgpu_vm_pt_ancestor(cursor)) { 474 cursor->pfn = ~0ll; 475 return; 476 } 477 } 478 } 479 480 /** 481 * amdgpu_vm_pt_first_dfs - start a deep first search 482 * 483 * @adev: amdgpu_device structure 484 * @vm: amdgpu_vm structure 485 * @cursor: state to initialize 486 * 487 * Starts a deep first traversal of the PD/PT tree. 488 */ 489 static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev, 490 struct amdgpu_vm *vm, 491 struct amdgpu_vm_pt_cursor *start, 492 struct amdgpu_vm_pt_cursor *cursor) 493 { 494 if (start) 495 *cursor = *start; 496 else 497 amdgpu_vm_pt_start(adev, vm, 0, cursor); 498 while (amdgpu_vm_pt_descendant(adev, cursor)); 499 } 500 501 /** 502 * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue 503 * 504 * @start: starting point for the search 505 * @entry: current entry 506 * 507 * Returns: 508 * True when the search should continue, false otherwise. 509 */ 510 static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start, 511 struct amdgpu_vm_pt *entry) 512 { 513 return entry && (!start || entry != start->entry); 514 } 515 516 /** 517 * amdgpu_vm_pt_next_dfs - get the next node for a deep first search 518 * 519 * @adev: amdgpu_device structure 520 * @cursor: current state 521 * 522 * Move the cursor to the next node in a deep first search. 523 */ 524 static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev, 525 struct amdgpu_vm_pt_cursor *cursor) 526 { 527 if (!cursor->entry) 528 return; 529 530 if (!cursor->parent) 531 cursor->entry = NULL; 532 else if (amdgpu_vm_pt_sibling(adev, cursor)) 533 while (amdgpu_vm_pt_descendant(adev, cursor)); 534 else 535 amdgpu_vm_pt_ancestor(cursor); 536 } 537 538 /** 539 * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs 540 */ 541 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \ 542 for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \ 543 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\ 544 amdgpu_vm_pt_continue_dfs((start), (entry)); \ 545 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor))) 546 547 /** 548 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list 549 * 550 * @vm: vm providing the BOs 551 * @validated: head of validation list 552 * @entry: entry to add 553 * 554 * Add the page directory to the list of BOs to 555 * validate for command submission. 556 */ 557 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, 558 struct list_head *validated, 559 struct amdgpu_bo_list_entry *entry) 560 { 561 entry->priority = 0; 562 entry->tv.bo = &vm->root.base.bo->tbo; 563 /* One for the VM updates, one for TTM and one for the CS job */ 564 entry->tv.num_shared = 3; 565 entry->user_pages = NULL; 566 list_add(&entry->tv.head, validated); 567 } 568 569 void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo) 570 { 571 struct amdgpu_bo *abo; 572 struct amdgpu_vm_bo_base *bo_base; 573 574 if (!amdgpu_bo_is_amdgpu_bo(bo)) 575 return; 576 577 if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) 578 return; 579 580 abo = ttm_to_amdgpu_bo(bo); 581 if (!abo->parent) 582 return; 583 for (bo_base = abo->vm_bo; bo_base; bo_base = bo_base->next) { 584 struct amdgpu_vm *vm = bo_base->vm; 585 586 if (abo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) 587 vm->bulk_moveable = false; 588 } 589 590 } 591 /** 592 * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU 593 * 594 * @adev: amdgpu device pointer 595 * @vm: vm providing the BOs 596 * 597 * Move all BOs to the end of LRU and remember their positions to put them 598 * together. 599 */ 600 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, 601 struct amdgpu_vm *vm) 602 { 603 struct ttm_bo_global *glob = adev->mman.bdev.glob; 604 struct amdgpu_vm_bo_base *bo_base; 605 606 #if 0 607 if (vm->bulk_moveable) { 608 spin_lock(&glob->lru_lock); 609 ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move); 610 spin_unlock(&glob->lru_lock); 611 return; 612 } 613 #endif 614 615 memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move)); 616 617 spin_lock(&glob->lru_lock); 618 list_for_each_entry(bo_base, &vm->idle, vm_status) { 619 struct amdgpu_bo *bo = bo_base->bo; 620 621 if (!bo->parent) 622 continue; 623 624 ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move); 625 if (bo->shadow) 626 ttm_bo_move_to_lru_tail(&bo->shadow->tbo, 627 &vm->lru_bulk_move); 628 } 629 spin_unlock(&glob->lru_lock); 630 631 vm->bulk_moveable = true; 632 } 633 634 /** 635 * amdgpu_vm_validate_pt_bos - validate the page table BOs 636 * 637 * @adev: amdgpu device pointer 638 * @vm: vm providing the BOs 639 * @validate: callback to do the validation 640 * @param: parameter for the validation callback 641 * 642 * Validate the page table BOs on command submission if neccessary. 643 * 644 * Returns: 645 * Validation result. 646 */ 647 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, 648 int (*validate)(void *p, struct amdgpu_bo *bo), 649 void *param) 650 { 651 struct amdgpu_vm_bo_base *bo_base, *tmp; 652 int r = 0; 653 654 vm->bulk_moveable &= list_empty(&vm->evicted); 655 656 list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) { 657 struct amdgpu_bo *bo = bo_base->bo; 658 659 r = validate(param, bo); 660 if (r) 661 break; 662 663 if (bo->tbo.type != ttm_bo_type_kernel) { 664 amdgpu_vm_bo_moved(bo_base); 665 } else { 666 vm->update_funcs->map_table(bo); 667 if (bo->parent) 668 amdgpu_vm_bo_relocated(bo_base); 669 else 670 amdgpu_vm_bo_idle(bo_base); 671 } 672 } 673 674 return r; 675 } 676 677 /** 678 * amdgpu_vm_ready - check VM is ready for updates 679 * 680 * @vm: VM to check 681 * 682 * Check if all VM PDs/PTs are ready for updates 683 * 684 * Returns: 685 * True if eviction list is empty. 686 */ 687 bool amdgpu_vm_ready(struct amdgpu_vm *vm) 688 { 689 return list_empty(&vm->evicted); 690 } 691 692 /** 693 * amdgpu_vm_clear_bo - initially clear the PDs/PTs 694 * 695 * @adev: amdgpu_device pointer 696 * @vm: VM to clear BO from 697 * @bo: BO to clear 698 * 699 * Root PD needs to be reserved when calling this. 700 * 701 * Returns: 702 * 0 on success, errno otherwise. 703 */ 704 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, 705 struct amdgpu_vm *vm, 706 struct amdgpu_bo *bo) 707 { 708 struct ttm_operation_ctx ctx = { true, false }; 709 unsigned level = adev->vm_manager.root_level; 710 struct amdgpu_vm_update_params params; 711 struct amdgpu_bo *ancestor = bo; 712 unsigned entries, ats_entries; 713 uint64_t addr; 714 int r; 715 716 /* Figure out our place in the hierarchy */ 717 if (ancestor->parent) { 718 ++level; 719 while (ancestor->parent->parent) { 720 ++level; 721 ancestor = ancestor->parent; 722 } 723 } 724 725 entries = amdgpu_bo_size(bo) / 8; 726 if (!vm->pte_support_ats) { 727 ats_entries = 0; 728 729 } else if (!bo->parent) { 730 ats_entries = amdgpu_vm_num_ats_entries(adev); 731 ats_entries = min(ats_entries, entries); 732 entries -= ats_entries; 733 734 } else { 735 struct amdgpu_vm_pt *pt; 736 737 pt = container_of(ancestor->vm_bo, struct amdgpu_vm_pt, base); 738 ats_entries = amdgpu_vm_num_ats_entries(adev); 739 if ((pt - vm->root.entries) >= ats_entries) { 740 ats_entries = 0; 741 } else { 742 ats_entries = entries; 743 entries = 0; 744 } 745 } 746 747 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 748 if (r) 749 return r; 750 751 if (bo->shadow) { 752 r = ttm_bo_validate(&bo->shadow->tbo, &bo->shadow->placement, 753 &ctx); 754 if (r) 755 return r; 756 } 757 758 r = vm->update_funcs->map_table(bo); 759 if (r) 760 return r; 761 762 memset(¶ms, 0, sizeof(params)); 763 params.adev = adev; 764 params.vm = vm; 765 766 r = vm->update_funcs->prepare(¶ms, AMDGPU_FENCE_OWNER_KFD, NULL); 767 if (r) 768 return r; 769 770 addr = 0; 771 if (ats_entries) { 772 uint64_t value = 0, flags; 773 774 flags = AMDGPU_PTE_DEFAULT_ATC; 775 if (level != AMDGPU_VM_PTB) { 776 /* Handle leaf PDEs as PTEs */ 777 flags |= AMDGPU_PDE_PTE; 778 amdgpu_gmc_get_vm_pde(adev, level, &value, &flags); 779 } 780 781 r = vm->update_funcs->update(¶ms, bo, addr, 0, ats_entries, 782 value, flags); 783 if (r) 784 return r; 785 786 addr += ats_entries * 8; 787 } 788 789 if (entries) { 790 uint64_t value = 0, flags = 0; 791 792 if (adev->asic_type >= CHIP_VEGA10) { 793 if (level != AMDGPU_VM_PTB) { 794 /* Handle leaf PDEs as PTEs */ 795 flags |= AMDGPU_PDE_PTE; 796 amdgpu_gmc_get_vm_pde(adev, level, 797 &value, &flags); 798 } else { 799 /* Workaround for fault priority problem on GMC9 */ 800 flags = AMDGPU_PTE_EXECUTABLE; 801 } 802 } 803 804 r = vm->update_funcs->update(¶ms, bo, addr, 0, entries, 805 value, flags); 806 if (r) 807 return r; 808 } 809 810 return vm->update_funcs->commit(¶ms, NULL); 811 } 812 813 /** 814 * amdgpu_vm_bo_param - fill in parameters for PD/PT allocation 815 * 816 * @adev: amdgpu_device pointer 817 * @vm: requesting vm 818 * @bp: resulting BO allocation parameters 819 */ 820 static void amdgpu_vm_bo_param(struct amdgpu_device *adev, struct amdgpu_vm *vm, 821 int level, struct amdgpu_bo_param *bp) 822 { 823 memset(bp, 0, sizeof(*bp)); 824 825 bp->size = amdgpu_vm_bo_size(adev, level); 826 bp->byte_align = AMDGPU_GPU_PAGE_SIZE; 827 bp->domain = AMDGPU_GEM_DOMAIN_VRAM; 828 bp->domain = amdgpu_bo_get_preferred_pin_domain(adev, bp->domain); 829 bp->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | 830 AMDGPU_GEM_CREATE_CPU_GTT_USWC; 831 if (vm->use_cpu_for_update) 832 bp->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 833 else if (!vm->root.base.bo || vm->root.base.bo->shadow) 834 bp->flags |= AMDGPU_GEM_CREATE_SHADOW; 835 bp->type = ttm_bo_type_kernel; 836 if (vm->root.base.bo) 837 bp->resv = vm->root.base.bo->tbo.base.resv; 838 } 839 840 /** 841 * amdgpu_vm_alloc_pts - Allocate a specific page table 842 * 843 * @adev: amdgpu_device pointer 844 * @vm: VM to allocate page tables for 845 * @cursor: Which page table to allocate 846 * 847 * Make sure a specific page table or directory is allocated. 848 * 849 * Returns: 850 * 1 if page table needed to be allocated, 0 if page table was already 851 * allocated, negative errno if an error occurred. 852 */ 853 static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, 854 struct amdgpu_vm *vm, 855 struct amdgpu_vm_pt_cursor *cursor) 856 { 857 struct amdgpu_vm_pt *entry = cursor->entry; 858 struct amdgpu_bo_param bp; 859 struct amdgpu_bo *pt; 860 int r; 861 862 if (cursor->level < AMDGPU_VM_PTB && !entry->entries) { 863 unsigned num_entries; 864 865 num_entries = amdgpu_vm_num_entries(adev, cursor->level); 866 entry->entries = kvmalloc_array(num_entries, 867 sizeof(*entry->entries), 868 GFP_KERNEL | __GFP_ZERO); 869 if (!entry->entries) 870 return -ENOMEM; 871 } 872 873 if (entry->base.bo) 874 return 0; 875 876 amdgpu_vm_bo_param(adev, vm, cursor->level, &bp); 877 878 r = amdgpu_bo_create(adev, &bp, &pt); 879 if (r) 880 return r; 881 882 /* Keep a reference to the root directory to avoid 883 * freeing them up in the wrong order. 884 */ 885 pt->parent = amdgpu_bo_ref(cursor->parent->base.bo); 886 amdgpu_vm_bo_base_init(&entry->base, vm, pt); 887 888 r = amdgpu_vm_clear_bo(adev, vm, pt); 889 if (r) 890 goto error_free_pt; 891 892 return 0; 893 894 error_free_pt: 895 amdgpu_bo_unref(&pt->shadow); 896 amdgpu_bo_unref(&pt); 897 return r; 898 } 899 900 /** 901 * amdgpu_vm_free_table - fre one PD/PT 902 * 903 * @entry: PDE to free 904 */ 905 static void amdgpu_vm_free_table(struct amdgpu_vm_pt *entry) 906 { 907 if (entry->base.bo) { 908 entry->base.bo->vm_bo = NULL; 909 list_del(&entry->base.vm_status); 910 amdgpu_bo_unref(&entry->base.bo->shadow); 911 amdgpu_bo_unref(&entry->base.bo); 912 } 913 kvfree(entry->entries); 914 entry->entries = NULL; 915 } 916 917 /** 918 * amdgpu_vm_free_pts - free PD/PT levels 919 * 920 * @adev: amdgpu device structure 921 * @vm: amdgpu vm structure 922 * @start: optional cursor where to start freeing PDs/PTs 923 * 924 * Free the page directory or page table level and all sub levels. 925 */ 926 static void amdgpu_vm_free_pts(struct amdgpu_device *adev, 927 struct amdgpu_vm *vm, 928 struct amdgpu_vm_pt_cursor *start) 929 { 930 struct amdgpu_vm_pt_cursor cursor; 931 struct amdgpu_vm_pt *entry; 932 933 vm->bulk_moveable = false; 934 935 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) 936 amdgpu_vm_free_table(entry); 937 938 if (start) 939 amdgpu_vm_free_table(start->entry); 940 } 941 942 /** 943 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug 944 * 945 * @adev: amdgpu_device pointer 946 */ 947 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev) 948 { 949 const struct amdgpu_ip_block *ip_block; 950 bool has_compute_vm_bug; 951 struct amdgpu_ring *ring; 952 int i; 953 954 has_compute_vm_bug = false; 955 956 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); 957 if (ip_block) { 958 /* Compute has a VM bug for GFX version < 7. 959 Compute has a VM bug for GFX 8 MEC firmware version < 673.*/ 960 if (ip_block->version->major <= 7) 961 has_compute_vm_bug = true; 962 else if (ip_block->version->major == 8) 963 if (adev->gfx.mec_fw_version < 673) 964 has_compute_vm_bug = true; 965 } 966 967 for (i = 0; i < adev->num_rings; i++) { 968 ring = adev->rings[i]; 969 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 970 /* only compute rings */ 971 ring->has_compute_vm_bug = has_compute_vm_bug; 972 else 973 ring->has_compute_vm_bug = false; 974 } 975 } 976 977 /** 978 * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job. 979 * 980 * @ring: ring on which the job will be submitted 981 * @job: job to submit 982 * 983 * Returns: 984 * True if sync is needed. 985 */ 986 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring, 987 struct amdgpu_job *job) 988 { 989 struct amdgpu_device *adev = ring->adev; 990 unsigned vmhub = ring->funcs->vmhub; 991 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 992 struct amdgpu_vmid *id; 993 bool gds_switch_needed; 994 bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug; 995 996 if (job->vmid == 0) 997 return false; 998 id = &id_mgr->ids[job->vmid]; 999 gds_switch_needed = ring->funcs->emit_gds_switch && ( 1000 id->gds_base != job->gds_base || 1001 id->gds_size != job->gds_size || 1002 id->gws_base != job->gws_base || 1003 id->gws_size != job->gws_size || 1004 id->oa_base != job->oa_base || 1005 id->oa_size != job->oa_size); 1006 1007 if (amdgpu_vmid_had_gpu_reset(adev, id)) 1008 return true; 1009 1010 return vm_flush_needed || gds_switch_needed; 1011 } 1012 1013 /** 1014 * amdgpu_vm_flush - hardware flush the vm 1015 * 1016 * @ring: ring to use for flush 1017 * @job: related job 1018 * @need_pipe_sync: is pipe sync needed 1019 * 1020 * Emit a VM flush when it is necessary. 1021 * 1022 * Returns: 1023 * 0 on success, errno otherwise. 1024 */ 1025 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync) 1026 { 1027 struct amdgpu_device *adev = ring->adev; 1028 unsigned vmhub = ring->funcs->vmhub; 1029 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 1030 struct amdgpu_vmid *id = &id_mgr->ids[job->vmid]; 1031 bool gds_switch_needed = ring->funcs->emit_gds_switch && ( 1032 id->gds_base != job->gds_base || 1033 id->gds_size != job->gds_size || 1034 id->gws_base != job->gws_base || 1035 id->gws_size != job->gws_size || 1036 id->oa_base != job->oa_base || 1037 id->oa_size != job->oa_size); 1038 bool vm_flush_needed = job->vm_needs_flush; 1039 bool pasid_mapping_needed = id->pasid != job->pasid || 1040 !id->pasid_mapping || 1041 !dma_fence_is_signaled(id->pasid_mapping); 1042 struct dma_fence *fence = NULL; 1043 unsigned patch_offset = 0; 1044 int r; 1045 1046 if (amdgpu_vmid_had_gpu_reset(adev, id)) { 1047 gds_switch_needed = true; 1048 vm_flush_needed = true; 1049 pasid_mapping_needed = true; 1050 } 1051 1052 gds_switch_needed &= !!ring->funcs->emit_gds_switch; 1053 vm_flush_needed &= !!ring->funcs->emit_vm_flush && 1054 job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET; 1055 pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping && 1056 ring->funcs->emit_wreg; 1057 1058 if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync) 1059 return 0; 1060 1061 if (ring->funcs->init_cond_exec) 1062 patch_offset = amdgpu_ring_init_cond_exec(ring); 1063 1064 if (need_pipe_sync) 1065 amdgpu_ring_emit_pipeline_sync(ring); 1066 1067 if (vm_flush_needed) { 1068 trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr); 1069 amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr); 1070 } 1071 1072 if (pasid_mapping_needed) 1073 amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid); 1074 1075 if (vm_flush_needed || pasid_mapping_needed) { 1076 r = amdgpu_fence_emit(ring, &fence, 0); 1077 if (r) 1078 return r; 1079 } 1080 1081 if (vm_flush_needed) { 1082 mutex_lock(&id_mgr->lock); 1083 dma_fence_put(id->last_flush); 1084 id->last_flush = dma_fence_get(fence); 1085 id->current_gpu_reset_count = 1086 atomic_read(&adev->gpu_reset_counter); 1087 mutex_unlock(&id_mgr->lock); 1088 } 1089 1090 if (pasid_mapping_needed) { 1091 id->pasid = job->pasid; 1092 dma_fence_put(id->pasid_mapping); 1093 id->pasid_mapping = dma_fence_get(fence); 1094 } 1095 dma_fence_put(fence); 1096 1097 if (ring->funcs->emit_gds_switch && gds_switch_needed) { 1098 id->gds_base = job->gds_base; 1099 id->gds_size = job->gds_size; 1100 id->gws_base = job->gws_base; 1101 id->gws_size = job->gws_size; 1102 id->oa_base = job->oa_base; 1103 id->oa_size = job->oa_size; 1104 amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base, 1105 job->gds_size, job->gws_base, 1106 job->gws_size, job->oa_base, 1107 job->oa_size); 1108 } 1109 1110 if (ring->funcs->patch_cond_exec) 1111 amdgpu_ring_patch_cond_exec(ring, patch_offset); 1112 1113 /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */ 1114 if (ring->funcs->emit_switch_buffer) { 1115 amdgpu_ring_emit_switch_buffer(ring); 1116 amdgpu_ring_emit_switch_buffer(ring); 1117 } 1118 return 0; 1119 } 1120 1121 /** 1122 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo 1123 * 1124 * @vm: requested vm 1125 * @bo: requested buffer object 1126 * 1127 * Find @bo inside the requested vm. 1128 * Search inside the @bos vm list for the requested vm 1129 * Returns the found bo_va or NULL if none is found 1130 * 1131 * Object has to be reserved! 1132 * 1133 * Returns: 1134 * Found bo_va or NULL. 1135 */ 1136 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, 1137 struct amdgpu_bo *bo) 1138 { 1139 struct amdgpu_vm_bo_base *base; 1140 1141 for (base = bo->vm_bo; base; base = base->next) { 1142 if (base->vm != vm) 1143 continue; 1144 1145 return container_of(base, struct amdgpu_bo_va, base); 1146 } 1147 return NULL; 1148 } 1149 1150 /** 1151 * amdgpu_vm_map_gart - Resolve gart mapping of addr 1152 * 1153 * @pages_addr: optional DMA address to use for lookup 1154 * @addr: the unmapped addr 1155 * 1156 * Look up the physical address of the page that the pte resolves 1157 * to. 1158 * 1159 * Returns: 1160 * The pointer for the page table entry. 1161 */ 1162 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr) 1163 { 1164 uint64_t result; 1165 1166 /* page table offset */ 1167 result = pages_addr[addr >> PAGE_SHIFT]; 1168 1169 /* in case cpu page size != gpu page size*/ 1170 result |= addr & (~PAGE_MASK); 1171 1172 result &= 0xFFFFFFFFFFFFF000ULL; 1173 1174 return result; 1175 } 1176 1177 /* 1178 * amdgpu_vm_update_pde - update a single level in the hierarchy 1179 * 1180 * @param: parameters for the update 1181 * @vm: requested vm 1182 * @entry: entry to update 1183 * 1184 * Makes sure the requested entry in parent is up to date. 1185 */ 1186 static int amdgpu_vm_update_pde(struct amdgpu_vm_update_params *params, 1187 struct amdgpu_vm *vm, 1188 struct amdgpu_vm_pt *entry) 1189 { 1190 struct amdgpu_vm_pt *parent = amdgpu_vm_pt_parent(entry); 1191 struct amdgpu_bo *bo = parent->base.bo, *pbo; 1192 uint64_t pde, pt, flags; 1193 unsigned level; 1194 1195 for (level = 0, pbo = bo->parent; pbo; ++level) 1196 pbo = pbo->parent; 1197 1198 level += params->adev->vm_manager.root_level; 1199 amdgpu_gmc_get_pde_for_bo(entry->base.bo, level, &pt, &flags); 1200 pde = (entry - parent->entries) * 8; 1201 return vm->update_funcs->update(params, bo, pde, pt, 1, 0, flags); 1202 } 1203 1204 /* 1205 * amdgpu_vm_invalidate_pds - mark all PDs as invalid 1206 * 1207 * @adev: amdgpu_device pointer 1208 * @vm: related vm 1209 * 1210 * Mark all PD level as invalid after an error. 1211 */ 1212 static void amdgpu_vm_invalidate_pds(struct amdgpu_device *adev, 1213 struct amdgpu_vm *vm) 1214 { 1215 struct amdgpu_vm_pt_cursor cursor; 1216 struct amdgpu_vm_pt *entry; 1217 1218 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry) 1219 if (entry->base.bo && !entry->base.moved) 1220 amdgpu_vm_bo_relocated(&entry->base); 1221 } 1222 1223 /* 1224 * amdgpu_vm_update_directories - make sure that all directories are valid 1225 * 1226 * @adev: amdgpu_device pointer 1227 * @vm: requested vm 1228 * 1229 * Makes sure all directories are up to date. 1230 * 1231 * Returns: 1232 * 0 for success, error for failure. 1233 */ 1234 int amdgpu_vm_update_directories(struct amdgpu_device *adev, 1235 struct amdgpu_vm *vm) 1236 { 1237 struct amdgpu_vm_update_params params; 1238 int r; 1239 1240 if (list_empty(&vm->relocated)) 1241 return 0; 1242 1243 memset(¶ms, 0, sizeof(params)); 1244 params.adev = adev; 1245 params.vm = vm; 1246 1247 r = vm->update_funcs->prepare(¶ms, AMDGPU_FENCE_OWNER_VM, NULL); 1248 if (r) 1249 return r; 1250 1251 while (!list_empty(&vm->relocated)) { 1252 struct amdgpu_vm_pt *entry; 1253 1254 entry = list_first_entry(&vm->relocated, struct amdgpu_vm_pt, 1255 base.vm_status); 1256 amdgpu_vm_bo_idle(&entry->base); 1257 1258 r = amdgpu_vm_update_pde(¶ms, vm, entry); 1259 if (r) 1260 goto error; 1261 } 1262 1263 r = vm->update_funcs->commit(¶ms, &vm->last_update); 1264 if (r) 1265 goto error; 1266 return 0; 1267 1268 error: 1269 amdgpu_vm_invalidate_pds(adev, vm); 1270 return r; 1271 } 1272 1273 /** 1274 * amdgpu_vm_update_flags - figure out flags for PTE updates 1275 * 1276 * Make sure to set the right flags for the PTEs at the desired level. 1277 */ 1278 static void amdgpu_vm_update_flags(struct amdgpu_vm_update_params *params, 1279 struct amdgpu_bo *bo, unsigned level, 1280 uint64_t pe, uint64_t addr, 1281 unsigned count, uint32_t incr, 1282 uint64_t flags) 1283 1284 { 1285 if (level != AMDGPU_VM_PTB) { 1286 flags |= AMDGPU_PDE_PTE; 1287 amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags); 1288 1289 } else if (params->adev->asic_type >= CHIP_VEGA10 && 1290 !(flags & AMDGPU_PTE_VALID) && 1291 !(flags & AMDGPU_PTE_PRT)) { 1292 1293 /* Workaround for fault priority problem on GMC9 */ 1294 flags |= AMDGPU_PTE_EXECUTABLE; 1295 } 1296 1297 params->vm->update_funcs->update(params, bo, pe, addr, count, incr, 1298 flags); 1299 } 1300 1301 /** 1302 * amdgpu_vm_fragment - get fragment for PTEs 1303 * 1304 * @params: see amdgpu_vm_update_params definition 1305 * @start: first PTE to handle 1306 * @end: last PTE to handle 1307 * @flags: hw mapping flags 1308 * @frag: resulting fragment size 1309 * @frag_end: end of this fragment 1310 * 1311 * Returns the first possible fragment for the start and end address. 1312 */ 1313 static void amdgpu_vm_fragment(struct amdgpu_vm_update_params *params, 1314 uint64_t start, uint64_t end, uint64_t flags, 1315 unsigned int *frag, uint64_t *frag_end) 1316 { 1317 /** 1318 * The MC L1 TLB supports variable sized pages, based on a fragment 1319 * field in the PTE. When this field is set to a non-zero value, page 1320 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE 1321 * flags are considered valid for all PTEs within the fragment range 1322 * and corresponding mappings are assumed to be physically contiguous. 1323 * 1324 * The L1 TLB can store a single PTE for the whole fragment, 1325 * significantly increasing the space available for translation 1326 * caching. This leads to large improvements in throughput when the 1327 * TLB is under pressure. 1328 * 1329 * The L2 TLB distributes small and large fragments into two 1330 * asymmetric partitions. The large fragment cache is significantly 1331 * larger. Thus, we try to use large fragments wherever possible. 1332 * Userspace can support this by aligning virtual base address and 1333 * allocation size to the fragment size. 1334 * 1335 * Starting with Vega10 the fragment size only controls the L1. The L2 1336 * is now directly feed with small/huge/giant pages from the walker. 1337 */ 1338 unsigned max_frag; 1339 1340 if (params->adev->asic_type < CHIP_VEGA10) 1341 max_frag = params->adev->vm_manager.fragment_size; 1342 else 1343 max_frag = 31; 1344 1345 /* system pages are non continuously */ 1346 if (params->pages_addr) { 1347 *frag = 0; 1348 *frag_end = end; 1349 return; 1350 } 1351 1352 /* This intentionally wraps around if no bit is set */ 1353 *frag = min((unsigned)ffs(start) - 1, (unsigned)fls64(end - start) - 1); 1354 if (*frag >= max_frag) { 1355 *frag = max_frag; 1356 *frag_end = end & ~((1ULL << max_frag) - 1); 1357 } else { 1358 *frag_end = start + (1 << *frag); 1359 } 1360 } 1361 1362 /** 1363 * amdgpu_vm_update_ptes - make sure that page tables are valid 1364 * 1365 * @params: see amdgpu_vm_update_params definition 1366 * @start: start of GPU address range 1367 * @end: end of GPU address range 1368 * @dst: destination address to map to, the next dst inside the function 1369 * @flags: mapping flags 1370 * 1371 * Update the page tables in the range @start - @end. 1372 * 1373 * Returns: 1374 * 0 for success, -EINVAL for failure. 1375 */ 1376 static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params, 1377 uint64_t start, uint64_t end, 1378 uint64_t dst, uint64_t flags) 1379 { 1380 struct amdgpu_device *adev = params->adev; 1381 struct amdgpu_vm_pt_cursor cursor; 1382 uint64_t frag_start = start, frag_end; 1383 unsigned int frag; 1384 int r; 1385 1386 /* figure out the initial fragment */ 1387 amdgpu_vm_fragment(params, frag_start, end, flags, &frag, &frag_end); 1388 1389 /* walk over the address space and update the PTs */ 1390 amdgpu_vm_pt_start(adev, params->vm, start, &cursor); 1391 while (cursor.pfn < end) { 1392 unsigned shift, parent_shift, mask; 1393 uint64_t incr, entry_end, pe_start; 1394 struct amdgpu_bo *pt; 1395 1396 r = amdgpu_vm_alloc_pts(params->adev, params->vm, &cursor); 1397 if (r) 1398 return r; 1399 1400 pt = cursor.entry->base.bo; 1401 1402 /* The root level can't be a huge page */ 1403 if (cursor.level == adev->vm_manager.root_level) { 1404 if (!amdgpu_vm_pt_descendant(adev, &cursor)) 1405 return -ENOENT; 1406 continue; 1407 } 1408 1409 shift = amdgpu_vm_level_shift(adev, cursor.level); 1410 parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1); 1411 if (adev->asic_type < CHIP_VEGA10 && 1412 (flags & AMDGPU_PTE_VALID)) { 1413 /* No huge page support before GMC v9 */ 1414 if (cursor.level != AMDGPU_VM_PTB) { 1415 if (!amdgpu_vm_pt_descendant(adev, &cursor)) 1416 return -ENOENT; 1417 continue; 1418 } 1419 } else if (frag < shift) { 1420 /* We can't use this level when the fragment size is 1421 * smaller than the address shift. Go to the next 1422 * child entry and try again. 1423 */ 1424 if (!amdgpu_vm_pt_descendant(adev, &cursor)) 1425 return -ENOENT; 1426 continue; 1427 } else if (frag >= parent_shift && 1428 cursor.level - 1 != adev->vm_manager.root_level) { 1429 /* If the fragment size is even larger than the parent 1430 * shift we should go up one level and check it again 1431 * unless one level up is the root level. 1432 */ 1433 if (!amdgpu_vm_pt_ancestor(&cursor)) 1434 return -ENOENT; 1435 continue; 1436 } 1437 1438 /* Looks good so far, calculate parameters for the update */ 1439 incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift; 1440 mask = amdgpu_vm_entries_mask(adev, cursor.level); 1441 pe_start = ((cursor.pfn >> shift) & mask) * 8; 1442 entry_end = (uint64_t)(mask + 1) << shift; 1443 entry_end += cursor.pfn & ~(entry_end - 1); 1444 entry_end = min(entry_end, end); 1445 1446 do { 1447 uint64_t upd_end = min(entry_end, frag_end); 1448 unsigned nptes = (upd_end - frag_start) >> shift; 1449 1450 amdgpu_vm_update_flags(params, pt, cursor.level, 1451 pe_start, dst, nptes, incr, 1452 flags | AMDGPU_PTE_FRAG(frag)); 1453 1454 pe_start += nptes * 8; 1455 dst += (uint64_t)nptes * AMDGPU_GPU_PAGE_SIZE << shift; 1456 1457 frag_start = upd_end; 1458 if (frag_start >= frag_end) { 1459 /* figure out the next fragment */ 1460 amdgpu_vm_fragment(params, frag_start, end, 1461 flags, &frag, &frag_end); 1462 if (frag < shift) 1463 break; 1464 } 1465 } while (frag_start < entry_end); 1466 1467 if (amdgpu_vm_pt_descendant(adev, &cursor)) { 1468 /* Free all child entries */ 1469 while (cursor.pfn < frag_start) { 1470 amdgpu_vm_free_pts(adev, params->vm, &cursor); 1471 amdgpu_vm_pt_next(adev, &cursor); 1472 } 1473 1474 } else if (frag >= shift) { 1475 /* or just move on to the next on the same level. */ 1476 amdgpu_vm_pt_next(adev, &cursor); 1477 } 1478 } 1479 1480 return 0; 1481 } 1482 1483 /** 1484 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table 1485 * 1486 * @adev: amdgpu_device pointer 1487 * @exclusive: fence we need to sync to 1488 * @pages_addr: DMA addresses to use for mapping 1489 * @vm: requested vm 1490 * @start: start of mapped range 1491 * @last: last mapped entry 1492 * @flags: flags for the entries 1493 * @addr: addr to set the area to 1494 * @fence: optional resulting fence 1495 * 1496 * Fill in the page table entries between @start and @last. 1497 * 1498 * Returns: 1499 * 0 for success, -EINVAL for failure. 1500 */ 1501 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, 1502 struct dma_fence *exclusive, 1503 dma_addr_t *pages_addr, 1504 struct amdgpu_vm *vm, 1505 uint64_t start, uint64_t last, 1506 uint64_t flags, uint64_t addr, 1507 struct dma_fence **fence) 1508 { 1509 struct amdgpu_vm_update_params params; 1510 void *owner = AMDGPU_FENCE_OWNER_VM; 1511 int r; 1512 1513 memset(¶ms, 0, sizeof(params)); 1514 params.adev = adev; 1515 params.vm = vm; 1516 params.pages_addr = pages_addr; 1517 1518 /* sync to everything except eviction fences on unmapping */ 1519 if (!(flags & AMDGPU_PTE_VALID)) 1520 owner = AMDGPU_FENCE_OWNER_KFD; 1521 1522 r = vm->update_funcs->prepare(¶ms, owner, exclusive); 1523 if (r) 1524 return r; 1525 1526 r = amdgpu_vm_update_ptes(¶ms, start, last + 1, addr, flags); 1527 if (r) 1528 return r; 1529 1530 return vm->update_funcs->commit(¶ms, fence); 1531 } 1532 1533 /** 1534 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks 1535 * 1536 * @adev: amdgpu_device pointer 1537 * @exclusive: fence we need to sync to 1538 * @pages_addr: DMA addresses to use for mapping 1539 * @vm: requested vm 1540 * @mapping: mapped range and flags to use for the update 1541 * @flags: HW flags for the mapping 1542 * @bo_adev: amdgpu_device pointer that bo actually been allocated 1543 * @nodes: array of drm_mm_nodes with the MC addresses 1544 * @fence: optional resulting fence 1545 * 1546 * Split the mapping into smaller chunks so that each update fits 1547 * into a SDMA IB. 1548 * 1549 * Returns: 1550 * 0 for success, -EINVAL for failure. 1551 */ 1552 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, 1553 struct dma_fence *exclusive, 1554 dma_addr_t *pages_addr, 1555 struct amdgpu_vm *vm, 1556 struct amdgpu_bo_va_mapping *mapping, 1557 uint64_t flags, 1558 struct amdgpu_device *bo_adev, 1559 struct drm_mm_node *nodes, 1560 struct dma_fence **fence) 1561 { 1562 unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size; 1563 uint64_t pfn, start = mapping->start; 1564 int r; 1565 1566 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here 1567 * but in case of something, we filter the flags in first place 1568 */ 1569 if (!(mapping->flags & AMDGPU_PTE_READABLE)) 1570 flags &= ~AMDGPU_PTE_READABLE; 1571 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) 1572 flags &= ~AMDGPU_PTE_WRITEABLE; 1573 1574 flags &= ~AMDGPU_PTE_EXECUTABLE; 1575 flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; 1576 1577 if (adev->asic_type >= CHIP_NAVI10) { 1578 flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK; 1579 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK); 1580 } else { 1581 flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; 1582 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK); 1583 } 1584 1585 if ((mapping->flags & AMDGPU_PTE_PRT) && 1586 (adev->asic_type >= CHIP_VEGA10)) { 1587 flags |= AMDGPU_PTE_PRT; 1588 if (adev->asic_type >= CHIP_NAVI10) { 1589 flags |= AMDGPU_PTE_SNOOPED; 1590 flags |= AMDGPU_PTE_LOG; 1591 flags |= AMDGPU_PTE_SYSTEM; 1592 } 1593 flags &= ~AMDGPU_PTE_VALID; 1594 } 1595 1596 trace_amdgpu_vm_bo_update(mapping); 1597 1598 pfn = mapping->offset >> PAGE_SHIFT; 1599 if (nodes) { 1600 while (pfn >= nodes->size) { 1601 pfn -= nodes->size; 1602 ++nodes; 1603 } 1604 } 1605 1606 do { 1607 dma_addr_t *dma_addr = NULL; 1608 uint64_t max_entries; 1609 uint64_t addr, last; 1610 1611 if (nodes) { 1612 addr = nodes->start << PAGE_SHIFT; 1613 max_entries = (nodes->size - pfn) * 1614 AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1615 } else { 1616 addr = 0; 1617 max_entries = S64_MAX; 1618 } 1619 1620 if (pages_addr) { 1621 uint64_t count; 1622 1623 for (count = 1; 1624 count < max_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1625 ++count) { 1626 uint64_t idx = pfn + count; 1627 1628 if (pages_addr[idx] != 1629 (pages_addr[idx - 1] + PAGE_SIZE)) 1630 break; 1631 } 1632 1633 if (count < min_linear_pages) { 1634 addr = pfn << PAGE_SHIFT; 1635 dma_addr = pages_addr; 1636 } else { 1637 addr = pages_addr[pfn]; 1638 max_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1639 } 1640 1641 } else if (flags & AMDGPU_PTE_VALID) { 1642 addr += bo_adev->vm_manager.vram_base_offset; 1643 addr += pfn << PAGE_SHIFT; 1644 } 1645 1646 last = min((uint64_t)mapping->last, start + max_entries - 1); 1647 r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm, 1648 start, last, flags, addr, 1649 fence); 1650 if (r) 1651 return r; 1652 1653 pfn += (last - start + 1) / AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1654 if (nodes && nodes->size == pfn) { 1655 pfn = 0; 1656 ++nodes; 1657 } 1658 start = last + 1; 1659 1660 } while (unlikely(start != mapping->last + 1)); 1661 1662 return 0; 1663 } 1664 1665 /** 1666 * amdgpu_vm_bo_update - update all BO mappings in the vm page table 1667 * 1668 * @adev: amdgpu_device pointer 1669 * @bo_va: requested BO and VM object 1670 * @clear: if true clear the entries 1671 * 1672 * Fill in the page table entries for @bo_va. 1673 * 1674 * Returns: 1675 * 0 for success, -EINVAL for failure. 1676 */ 1677 int amdgpu_vm_bo_update(struct amdgpu_device *adev, 1678 struct amdgpu_bo_va *bo_va, 1679 bool clear) 1680 { 1681 struct amdgpu_bo *bo = bo_va->base.bo; 1682 struct amdgpu_vm *vm = bo_va->base.vm; 1683 struct amdgpu_bo_va_mapping *mapping; 1684 dma_addr_t *pages_addr = NULL; 1685 struct ttm_mem_reg *mem; 1686 struct drm_mm_node *nodes; 1687 struct dma_fence *exclusive, **last_update; 1688 uint64_t flags; 1689 struct amdgpu_device *bo_adev = adev; 1690 int r; 1691 1692 if (clear || !bo) { 1693 mem = NULL; 1694 nodes = NULL; 1695 exclusive = NULL; 1696 } else { 1697 struct ttm_dma_tt *ttm; 1698 1699 mem = &bo->tbo.mem; 1700 nodes = mem->mm_node; 1701 if (mem->mem_type == TTM_PL_TT) { 1702 ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm); 1703 pages_addr = ttm->dma_address; 1704 } 1705 exclusive = dma_resv_get_excl(bo->tbo.base.resv); 1706 } 1707 1708 if (bo) { 1709 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem); 1710 bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); 1711 } else { 1712 flags = 0x0; 1713 } 1714 1715 if (clear || (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv)) 1716 last_update = &vm->last_update; 1717 else 1718 last_update = &bo_va->last_pt_update; 1719 1720 if (!clear && bo_va->base.moved) { 1721 bo_va->base.moved = false; 1722 list_splice_init(&bo_va->valids, &bo_va->invalids); 1723 1724 } else if (bo_va->cleared != clear) { 1725 list_splice_init(&bo_va->valids, &bo_va->invalids); 1726 } 1727 1728 list_for_each_entry(mapping, &bo_va->invalids, list) { 1729 r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm, 1730 mapping, flags, bo_adev, nodes, 1731 last_update); 1732 if (r) 1733 return r; 1734 } 1735 1736 if (vm->use_cpu_for_update) { 1737 /* Flush HDP */ 1738 mb(); 1739 amdgpu_asic_flush_hdp(adev, NULL); 1740 } 1741 1742 /* If the BO is not in its preferred location add it back to 1743 * the evicted list so that it gets validated again on the 1744 * next command submission. 1745 */ 1746 if (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) { 1747 uint32_t mem_type = bo->tbo.mem.mem_type; 1748 1749 if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type))) 1750 amdgpu_vm_bo_evicted(&bo_va->base); 1751 else 1752 amdgpu_vm_bo_idle(&bo_va->base); 1753 } else { 1754 amdgpu_vm_bo_done(&bo_va->base); 1755 } 1756 1757 list_splice_init(&bo_va->invalids, &bo_va->valids); 1758 bo_va->cleared = clear; 1759 1760 if (trace_amdgpu_vm_bo_mapping_enabled()) { 1761 list_for_each_entry(mapping, &bo_va->valids, list) 1762 trace_amdgpu_vm_bo_mapping(mapping); 1763 } 1764 1765 return 0; 1766 } 1767 1768 /** 1769 * amdgpu_vm_update_prt_state - update the global PRT state 1770 * 1771 * @adev: amdgpu_device pointer 1772 */ 1773 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev) 1774 { 1775 unsigned long flags; 1776 bool enable; 1777 1778 spin_lock_irqsave(&adev->vm_manager.prt_lock, flags); 1779 enable = !!atomic_read(&adev->vm_manager.num_prt_users); 1780 adev->gmc.gmc_funcs->set_prt(adev, enable); 1781 spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags); 1782 } 1783 1784 /** 1785 * amdgpu_vm_prt_get - add a PRT user 1786 * 1787 * @adev: amdgpu_device pointer 1788 */ 1789 static void amdgpu_vm_prt_get(struct amdgpu_device *adev) 1790 { 1791 if (!adev->gmc.gmc_funcs->set_prt) 1792 return; 1793 1794 if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1) 1795 amdgpu_vm_update_prt_state(adev); 1796 } 1797 1798 /** 1799 * amdgpu_vm_prt_put - drop a PRT user 1800 * 1801 * @adev: amdgpu_device pointer 1802 */ 1803 static void amdgpu_vm_prt_put(struct amdgpu_device *adev) 1804 { 1805 if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0) 1806 amdgpu_vm_update_prt_state(adev); 1807 } 1808 1809 /** 1810 * amdgpu_vm_prt_cb - callback for updating the PRT status 1811 * 1812 * @fence: fence for the callback 1813 * @_cb: the callback function 1814 */ 1815 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb) 1816 { 1817 struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb); 1818 1819 amdgpu_vm_prt_put(cb->adev); 1820 kfree(cb); 1821 } 1822 1823 /** 1824 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status 1825 * 1826 * @adev: amdgpu_device pointer 1827 * @fence: fence for the callback 1828 */ 1829 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev, 1830 struct dma_fence *fence) 1831 { 1832 struct amdgpu_prt_cb *cb; 1833 1834 if (!adev->gmc.gmc_funcs->set_prt) 1835 return; 1836 1837 cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL); 1838 if (!cb) { 1839 /* Last resort when we are OOM */ 1840 if (fence) 1841 dma_fence_wait(fence, false); 1842 1843 amdgpu_vm_prt_put(adev); 1844 } else { 1845 cb->adev = adev; 1846 if (!fence || dma_fence_add_callback(fence, &cb->cb, 1847 amdgpu_vm_prt_cb)) 1848 amdgpu_vm_prt_cb(fence, &cb->cb); 1849 } 1850 } 1851 1852 /** 1853 * amdgpu_vm_free_mapping - free a mapping 1854 * 1855 * @adev: amdgpu_device pointer 1856 * @vm: requested vm 1857 * @mapping: mapping to be freed 1858 * @fence: fence of the unmap operation 1859 * 1860 * Free a mapping and make sure we decrease the PRT usage count if applicable. 1861 */ 1862 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev, 1863 struct amdgpu_vm *vm, 1864 struct amdgpu_bo_va_mapping *mapping, 1865 struct dma_fence *fence) 1866 { 1867 if (mapping->flags & AMDGPU_PTE_PRT) 1868 amdgpu_vm_add_prt_cb(adev, fence); 1869 kfree(mapping); 1870 } 1871 1872 /** 1873 * amdgpu_vm_prt_fini - finish all prt mappings 1874 * 1875 * @adev: amdgpu_device pointer 1876 * @vm: requested vm 1877 * 1878 * Register a cleanup callback to disable PRT support after VM dies. 1879 */ 1880 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1881 { 1882 struct dma_resv *resv = vm->root.base.bo->tbo.base.resv; 1883 struct dma_fence *excl, **shared; 1884 unsigned i, shared_count; 1885 int r; 1886 1887 r = dma_resv_get_fences_rcu(resv, &excl, 1888 &shared_count, &shared); 1889 if (r) { 1890 /* Not enough memory to grab the fence list, as last resort 1891 * block for all the fences to complete. 1892 */ 1893 dma_resv_wait_timeout_rcu(resv, true, false, 1894 MAX_SCHEDULE_TIMEOUT); 1895 return; 1896 } 1897 1898 /* Add a callback for each fence in the reservation object */ 1899 amdgpu_vm_prt_get(adev); 1900 amdgpu_vm_add_prt_cb(adev, excl); 1901 1902 for (i = 0; i < shared_count; ++i) { 1903 amdgpu_vm_prt_get(adev); 1904 amdgpu_vm_add_prt_cb(adev, shared[i]); 1905 } 1906 1907 kfree(shared); 1908 } 1909 1910 /** 1911 * amdgpu_vm_clear_freed - clear freed BOs in the PT 1912 * 1913 * @adev: amdgpu_device pointer 1914 * @vm: requested vm 1915 * @fence: optional resulting fence (unchanged if no work needed to be done 1916 * or if an error occurred) 1917 * 1918 * Make sure all freed BOs are cleared in the PT. 1919 * PTs have to be reserved and mutex must be locked! 1920 * 1921 * Returns: 1922 * 0 for success. 1923 * 1924 */ 1925 int amdgpu_vm_clear_freed(struct amdgpu_device *adev, 1926 struct amdgpu_vm *vm, 1927 struct dma_fence **fence) 1928 { 1929 struct amdgpu_bo_va_mapping *mapping; 1930 uint64_t init_pte_value = 0; 1931 struct dma_fence *f = NULL; 1932 int r; 1933 1934 while (!list_empty(&vm->freed)) { 1935 mapping = list_first_entry(&vm->freed, 1936 struct amdgpu_bo_va_mapping, list); 1937 list_del(&mapping->list); 1938 1939 if (vm->pte_support_ats && 1940 mapping->start < AMDGPU_GMC_HOLE_START) 1941 init_pte_value = AMDGPU_PTE_DEFAULT_ATC; 1942 1943 r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm, 1944 mapping->start, mapping->last, 1945 init_pte_value, 0, &f); 1946 amdgpu_vm_free_mapping(adev, vm, mapping, f); 1947 if (r) { 1948 dma_fence_put(f); 1949 return r; 1950 } 1951 } 1952 1953 if (fence && f) { 1954 dma_fence_put(*fence); 1955 *fence = f; 1956 } else { 1957 dma_fence_put(f); 1958 } 1959 1960 return 0; 1961 1962 } 1963 1964 /** 1965 * amdgpu_vm_handle_moved - handle moved BOs in the PT 1966 * 1967 * @adev: amdgpu_device pointer 1968 * @vm: requested vm 1969 * 1970 * Make sure all BOs which are moved are updated in the PTs. 1971 * 1972 * Returns: 1973 * 0 for success. 1974 * 1975 * PTs have to be reserved! 1976 */ 1977 int amdgpu_vm_handle_moved(struct amdgpu_device *adev, 1978 struct amdgpu_vm *vm) 1979 { 1980 struct amdgpu_bo_va *bo_va, *tmp; 1981 struct dma_resv *resv; 1982 bool clear; 1983 int r; 1984 1985 list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { 1986 /* Per VM BOs never need to bo cleared in the page tables */ 1987 r = amdgpu_vm_bo_update(adev, bo_va, false); 1988 if (r) 1989 return r; 1990 } 1991 1992 spin_lock(&vm->invalidated_lock); 1993 while (!list_empty(&vm->invalidated)) { 1994 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, 1995 base.vm_status); 1996 resv = bo_va->base.bo->tbo.base.resv; 1997 spin_unlock(&vm->invalidated_lock); 1998 1999 /* Try to reserve the BO to avoid clearing its ptes */ 2000 if (!amdgpu_vm_debug && dma_resv_trylock(resv)) 2001 clear = false; 2002 /* Somebody else is using the BO right now */ 2003 else 2004 clear = true; 2005 2006 r = amdgpu_vm_bo_update(adev, bo_va, clear); 2007 if (r) 2008 return r; 2009 2010 if (!clear) 2011 dma_resv_unlock(resv); 2012 spin_lock(&vm->invalidated_lock); 2013 } 2014 spin_unlock(&vm->invalidated_lock); 2015 2016 return 0; 2017 } 2018 2019 /** 2020 * amdgpu_vm_bo_add - add a bo to a specific vm 2021 * 2022 * @adev: amdgpu_device pointer 2023 * @vm: requested vm 2024 * @bo: amdgpu buffer object 2025 * 2026 * Add @bo into the requested vm. 2027 * Add @bo to the list of bos associated with the vm 2028 * 2029 * Returns: 2030 * Newly added bo_va or NULL for failure 2031 * 2032 * Object has to be reserved! 2033 */ 2034 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, 2035 struct amdgpu_vm *vm, 2036 struct amdgpu_bo *bo) 2037 { 2038 struct amdgpu_bo_va *bo_va; 2039 2040 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL); 2041 if (bo_va == NULL) { 2042 return NULL; 2043 } 2044 amdgpu_vm_bo_base_init(&bo_va->base, vm, bo); 2045 2046 bo_va->ref_count = 1; 2047 INIT_LIST_HEAD(&bo_va->valids); 2048 INIT_LIST_HEAD(&bo_va->invalids); 2049 2050 if (bo && amdgpu_xgmi_same_hive(adev, amdgpu_ttm_adev(bo->tbo.bdev)) && 2051 (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) { 2052 bo_va->is_xgmi = true; 2053 mutex_lock(&adev->vm_manager.lock_pstate); 2054 /* Power up XGMI if it can be potentially used */ 2055 if (++adev->vm_manager.xgmi_map_counter == 1) 2056 amdgpu_xgmi_set_pstate(adev, 1); 2057 mutex_unlock(&adev->vm_manager.lock_pstate); 2058 } 2059 2060 return bo_va; 2061 } 2062 2063 2064 /** 2065 * amdgpu_vm_bo_insert_mapping - insert a new mapping 2066 * 2067 * @adev: amdgpu_device pointer 2068 * @bo_va: bo_va to store the address 2069 * @mapping: the mapping to insert 2070 * 2071 * Insert a new mapping into all structures. 2072 */ 2073 static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev, 2074 struct amdgpu_bo_va *bo_va, 2075 struct amdgpu_bo_va_mapping *mapping) 2076 { 2077 struct amdgpu_vm *vm = bo_va->base.vm; 2078 struct amdgpu_bo *bo = bo_va->base.bo; 2079 2080 mapping->bo_va = bo_va; 2081 list_add(&mapping->list, &bo_va->invalids); 2082 amdgpu_vm_it_insert(mapping, &vm->va); 2083 2084 if (mapping->flags & AMDGPU_PTE_PRT) 2085 amdgpu_vm_prt_get(adev); 2086 2087 if (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv && 2088 !bo_va->base.moved) { 2089 list_move(&bo_va->base.vm_status, &vm->moved); 2090 } 2091 trace_amdgpu_vm_bo_map(bo_va, mapping); 2092 } 2093 2094 /** 2095 * amdgpu_vm_bo_map - map bo inside a vm 2096 * 2097 * @adev: amdgpu_device pointer 2098 * @bo_va: bo_va to store the address 2099 * @saddr: where to map the BO 2100 * @offset: requested offset in the BO 2101 * @size: BO size in bytes 2102 * @flags: attributes of pages (read/write/valid/etc.) 2103 * 2104 * Add a mapping of the BO at the specefied addr into the VM. 2105 * 2106 * Returns: 2107 * 0 for success, error for failure. 2108 * 2109 * Object has to be reserved and unreserved outside! 2110 */ 2111 int amdgpu_vm_bo_map(struct amdgpu_device *adev, 2112 struct amdgpu_bo_va *bo_va, 2113 uint64_t saddr, uint64_t offset, 2114 uint64_t size, uint64_t flags) 2115 { 2116 struct amdgpu_bo_va_mapping *mapping, *tmp; 2117 struct amdgpu_bo *bo = bo_va->base.bo; 2118 struct amdgpu_vm *vm = bo_va->base.vm; 2119 uint64_t eaddr; 2120 2121 /* validate the parameters */ 2122 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || 2123 size == 0 || size & AMDGPU_GPU_PAGE_MASK) 2124 return -EINVAL; 2125 2126 /* make sure object fit at this offset */ 2127 eaddr = saddr + size - 1; 2128 if (saddr >= eaddr || 2129 (bo && offset + size > amdgpu_bo_size(bo))) 2130 return -EINVAL; 2131 2132 saddr /= AMDGPU_GPU_PAGE_SIZE; 2133 eaddr /= AMDGPU_GPU_PAGE_SIZE; 2134 2135 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); 2136 if (tmp) { 2137 /* bo and tmp overlap, invalid addr */ 2138 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with " 2139 "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr, 2140 tmp->start, tmp->last + 1); 2141 return -EINVAL; 2142 } 2143 2144 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 2145 if (!mapping) 2146 return -ENOMEM; 2147 2148 mapping->start = saddr; 2149 mapping->last = eaddr; 2150 mapping->offset = offset; 2151 mapping->flags = flags; 2152 2153 amdgpu_vm_bo_insert_map(adev, bo_va, mapping); 2154 2155 return 0; 2156 } 2157 2158 /** 2159 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings 2160 * 2161 * @adev: amdgpu_device pointer 2162 * @bo_va: bo_va to store the address 2163 * @saddr: where to map the BO 2164 * @offset: requested offset in the BO 2165 * @size: BO size in bytes 2166 * @flags: attributes of pages (read/write/valid/etc.) 2167 * 2168 * Add a mapping of the BO at the specefied addr into the VM. Replace existing 2169 * mappings as we do so. 2170 * 2171 * Returns: 2172 * 0 for success, error for failure. 2173 * 2174 * Object has to be reserved and unreserved outside! 2175 */ 2176 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev, 2177 struct amdgpu_bo_va *bo_va, 2178 uint64_t saddr, uint64_t offset, 2179 uint64_t size, uint64_t flags) 2180 { 2181 struct amdgpu_bo_va_mapping *mapping; 2182 struct amdgpu_bo *bo = bo_va->base.bo; 2183 uint64_t eaddr; 2184 int r; 2185 2186 /* validate the parameters */ 2187 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || 2188 size == 0 || size & AMDGPU_GPU_PAGE_MASK) 2189 return -EINVAL; 2190 2191 /* make sure object fit at this offset */ 2192 eaddr = saddr + size - 1; 2193 if (saddr >= eaddr || 2194 (bo && offset + size > amdgpu_bo_size(bo))) 2195 return -EINVAL; 2196 2197 /* Allocate all the needed memory */ 2198 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 2199 if (!mapping) 2200 return -ENOMEM; 2201 2202 r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size); 2203 if (r) { 2204 kfree(mapping); 2205 return r; 2206 } 2207 2208 saddr /= AMDGPU_GPU_PAGE_SIZE; 2209 eaddr /= AMDGPU_GPU_PAGE_SIZE; 2210 2211 mapping->start = saddr; 2212 mapping->last = eaddr; 2213 mapping->offset = offset; 2214 mapping->flags = flags; 2215 2216 amdgpu_vm_bo_insert_map(adev, bo_va, mapping); 2217 2218 return 0; 2219 } 2220 2221 /** 2222 * amdgpu_vm_bo_unmap - remove bo mapping from vm 2223 * 2224 * @adev: amdgpu_device pointer 2225 * @bo_va: bo_va to remove the address from 2226 * @saddr: where to the BO is mapped 2227 * 2228 * Remove a mapping of the BO at the specefied addr from the VM. 2229 * 2230 * Returns: 2231 * 0 for success, error for failure. 2232 * 2233 * Object has to be reserved and unreserved outside! 2234 */ 2235 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, 2236 struct amdgpu_bo_va *bo_va, 2237 uint64_t saddr) 2238 { 2239 struct amdgpu_bo_va_mapping *mapping; 2240 struct amdgpu_vm *vm = bo_va->base.vm; 2241 bool valid = true; 2242 2243 saddr /= AMDGPU_GPU_PAGE_SIZE; 2244 2245 list_for_each_entry(mapping, &bo_va->valids, list) { 2246 if (mapping->start == saddr) 2247 break; 2248 } 2249 2250 if (&mapping->list == &bo_va->valids) { 2251 valid = false; 2252 2253 list_for_each_entry(mapping, &bo_va->invalids, list) { 2254 if (mapping->start == saddr) 2255 break; 2256 } 2257 2258 if (&mapping->list == &bo_va->invalids) 2259 return -ENOENT; 2260 } 2261 2262 list_del(&mapping->list); 2263 amdgpu_vm_it_remove(mapping, &vm->va); 2264 mapping->bo_va = NULL; 2265 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 2266 2267 if (valid) 2268 list_add(&mapping->list, &vm->freed); 2269 else 2270 amdgpu_vm_free_mapping(adev, vm, mapping, 2271 bo_va->last_pt_update); 2272 2273 return 0; 2274 } 2275 2276 /** 2277 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range 2278 * 2279 * @adev: amdgpu_device pointer 2280 * @vm: VM structure to use 2281 * @saddr: start of the range 2282 * @size: size of the range 2283 * 2284 * Remove all mappings in a range, split them as appropriate. 2285 * 2286 * Returns: 2287 * 0 for success, error for failure. 2288 */ 2289 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, 2290 struct amdgpu_vm *vm, 2291 uint64_t saddr, uint64_t size) 2292 { 2293 struct amdgpu_bo_va_mapping *before, *after, *tmp, *next; 2294 LIST_HEAD(removed); 2295 uint64_t eaddr; 2296 2297 eaddr = saddr + size - 1; 2298 saddr /= AMDGPU_GPU_PAGE_SIZE; 2299 eaddr /= AMDGPU_GPU_PAGE_SIZE; 2300 2301 /* Allocate all the needed memory */ 2302 before = kzalloc(sizeof(*before), GFP_KERNEL); 2303 if (!before) 2304 return -ENOMEM; 2305 INIT_LIST_HEAD(&before->list); 2306 2307 after = kzalloc(sizeof(*after), GFP_KERNEL); 2308 if (!after) { 2309 kfree(before); 2310 return -ENOMEM; 2311 } 2312 INIT_LIST_HEAD(&after->list); 2313 2314 /* Now gather all removed mappings */ 2315 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); 2316 while (tmp) { 2317 /* Remember mapping split at the start */ 2318 if (tmp->start < saddr) { 2319 before->start = tmp->start; 2320 before->last = saddr - 1; 2321 before->offset = tmp->offset; 2322 before->flags = tmp->flags; 2323 before->bo_va = tmp->bo_va; 2324 list_add(&before->list, &tmp->bo_va->invalids); 2325 } 2326 2327 /* Remember mapping split at the end */ 2328 if (tmp->last > eaddr) { 2329 after->start = eaddr + 1; 2330 after->last = tmp->last; 2331 after->offset = tmp->offset; 2332 after->offset += after->start - tmp->start; 2333 after->flags = tmp->flags; 2334 after->bo_va = tmp->bo_va; 2335 list_add(&after->list, &tmp->bo_va->invalids); 2336 } 2337 2338 list_del(&tmp->list); 2339 list_add(&tmp->list, &removed); 2340 2341 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr); 2342 } 2343 2344 /* And free them up */ 2345 list_for_each_entry_safe(tmp, next, &removed, list) { 2346 amdgpu_vm_it_remove(tmp, &vm->va); 2347 list_del(&tmp->list); 2348 2349 if (tmp->start < saddr) 2350 tmp->start = saddr; 2351 if (tmp->last > eaddr) 2352 tmp->last = eaddr; 2353 2354 tmp->bo_va = NULL; 2355 list_add(&tmp->list, &vm->freed); 2356 trace_amdgpu_vm_bo_unmap(NULL, tmp); 2357 } 2358 2359 /* Insert partial mapping before the range */ 2360 if (!list_empty(&before->list)) { 2361 amdgpu_vm_it_insert(before, &vm->va); 2362 if (before->flags & AMDGPU_PTE_PRT) 2363 amdgpu_vm_prt_get(adev); 2364 } else { 2365 kfree(before); 2366 } 2367 2368 /* Insert partial mapping after the range */ 2369 if (!list_empty(&after->list)) { 2370 amdgpu_vm_it_insert(after, &vm->va); 2371 if (after->flags & AMDGPU_PTE_PRT) 2372 amdgpu_vm_prt_get(adev); 2373 } else { 2374 kfree(after); 2375 } 2376 2377 return 0; 2378 } 2379 2380 /** 2381 * amdgpu_vm_bo_lookup_mapping - find mapping by address 2382 * 2383 * @vm: the requested VM 2384 * @addr: the address 2385 * 2386 * Find a mapping by it's address. 2387 * 2388 * Returns: 2389 * The amdgpu_bo_va_mapping matching for addr or NULL 2390 * 2391 */ 2392 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm, 2393 uint64_t addr) 2394 { 2395 return amdgpu_vm_it_iter_first(&vm->va, addr, addr); 2396 } 2397 2398 /** 2399 * amdgpu_vm_bo_trace_cs - trace all reserved mappings 2400 * 2401 * @vm: the requested vm 2402 * @ticket: CS ticket 2403 * 2404 * Trace all mappings of BOs reserved during a command submission. 2405 */ 2406 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket) 2407 { 2408 struct amdgpu_bo_va_mapping *mapping; 2409 2410 if (!trace_amdgpu_vm_bo_cs_enabled()) 2411 return; 2412 2413 for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping; 2414 mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) { 2415 if (mapping->bo_va && mapping->bo_va->base.bo) { 2416 struct amdgpu_bo *bo; 2417 2418 bo = mapping->bo_va->base.bo; 2419 if (dma_resv_locking_ctx(bo->tbo.base.resv) != 2420 ticket) 2421 continue; 2422 } 2423 2424 trace_amdgpu_vm_bo_cs(mapping); 2425 } 2426 } 2427 2428 /** 2429 * amdgpu_vm_bo_rmv - remove a bo to a specific vm 2430 * 2431 * @adev: amdgpu_device pointer 2432 * @bo_va: requested bo_va 2433 * 2434 * Remove @bo_va->bo from the requested vm. 2435 * 2436 * Object have to be reserved! 2437 */ 2438 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, 2439 struct amdgpu_bo_va *bo_va) 2440 { 2441 struct amdgpu_bo_va_mapping *mapping, *next; 2442 struct amdgpu_bo *bo = bo_va->base.bo; 2443 struct amdgpu_vm *vm = bo_va->base.vm; 2444 struct amdgpu_vm_bo_base **base; 2445 2446 if (bo) { 2447 if (bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) 2448 vm->bulk_moveable = false; 2449 2450 for (base = &bo_va->base.bo->vm_bo; *base; 2451 base = &(*base)->next) { 2452 if (*base != &bo_va->base) 2453 continue; 2454 2455 *base = bo_va->base.next; 2456 break; 2457 } 2458 } 2459 2460 spin_lock(&vm->invalidated_lock); 2461 list_del(&bo_va->base.vm_status); 2462 spin_unlock(&vm->invalidated_lock); 2463 2464 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 2465 list_del(&mapping->list); 2466 amdgpu_vm_it_remove(mapping, &vm->va); 2467 mapping->bo_va = NULL; 2468 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 2469 list_add(&mapping->list, &vm->freed); 2470 } 2471 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { 2472 list_del(&mapping->list); 2473 amdgpu_vm_it_remove(mapping, &vm->va); 2474 amdgpu_vm_free_mapping(adev, vm, mapping, 2475 bo_va->last_pt_update); 2476 } 2477 2478 dma_fence_put(bo_va->last_pt_update); 2479 2480 if (bo && bo_va->is_xgmi) { 2481 mutex_lock(&adev->vm_manager.lock_pstate); 2482 if (--adev->vm_manager.xgmi_map_counter == 0) 2483 amdgpu_xgmi_set_pstate(adev, 0); 2484 mutex_unlock(&adev->vm_manager.lock_pstate); 2485 } 2486 2487 kfree(bo_va); 2488 } 2489 2490 /** 2491 * amdgpu_vm_bo_invalidate - mark the bo as invalid 2492 * 2493 * @adev: amdgpu_device pointer 2494 * @bo: amdgpu buffer object 2495 * @evicted: is the BO evicted 2496 * 2497 * Mark @bo as invalid. 2498 */ 2499 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, 2500 struct amdgpu_bo *bo, bool evicted) 2501 { 2502 struct amdgpu_vm_bo_base *bo_base; 2503 2504 /* shadow bo doesn't have bo base, its validation needs its parent */ 2505 if (bo->parent && bo->parent->shadow == bo) 2506 bo = bo->parent; 2507 2508 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { 2509 struct amdgpu_vm *vm = bo_base->vm; 2510 2511 if (evicted && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) { 2512 amdgpu_vm_bo_evicted(bo_base); 2513 continue; 2514 } 2515 2516 if (bo_base->moved) 2517 continue; 2518 bo_base->moved = true; 2519 2520 if (bo->tbo.type == ttm_bo_type_kernel) 2521 amdgpu_vm_bo_relocated(bo_base); 2522 else if (bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) 2523 amdgpu_vm_bo_moved(bo_base); 2524 else 2525 amdgpu_vm_bo_invalidated(bo_base); 2526 } 2527 } 2528 2529 /** 2530 * amdgpu_vm_get_block_size - calculate VM page table size as power of two 2531 * 2532 * @vm_size: VM size 2533 * 2534 * Returns: 2535 * VM page table as power of two 2536 */ 2537 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size) 2538 { 2539 /* Total bits covered by PD + PTs */ 2540 unsigned bits = ilog2(vm_size) + 18; 2541 2542 /* Make sure the PD is 4K in size up to 8GB address space. 2543 Above that split equal between PD and PTs */ 2544 if (vm_size <= 8) 2545 return (bits - 9); 2546 else 2547 return ((bits + 3) / 2); 2548 } 2549 2550 /** 2551 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size 2552 * 2553 * @adev: amdgpu_device pointer 2554 * @min_vm_size: the minimum vm size in GB if it's set auto 2555 * @fragment_size_default: Default PTE fragment size 2556 * @max_level: max VMPT level 2557 * @max_bits: max address space size in bits 2558 * 2559 */ 2560 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size, 2561 uint32_t fragment_size_default, unsigned max_level, 2562 unsigned max_bits) 2563 { 2564 unsigned int max_size = 1 << (max_bits - 30); 2565 unsigned int vm_size; 2566 uint64_t tmp; 2567 2568 /* adjust vm size first */ 2569 if (amdgpu_vm_size != -1) { 2570 vm_size = amdgpu_vm_size; 2571 if (vm_size > max_size) { 2572 dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n", 2573 amdgpu_vm_size, max_size); 2574 vm_size = max_size; 2575 } 2576 } else { 2577 struct sysinfo si; 2578 unsigned int phys_ram_gb; 2579 2580 /* Optimal VM size depends on the amount of physical 2581 * RAM available. Underlying requirements and 2582 * assumptions: 2583 * 2584 * - Need to map system memory and VRAM from all GPUs 2585 * - VRAM from other GPUs not known here 2586 * - Assume VRAM <= system memory 2587 * - On GFX8 and older, VM space can be segmented for 2588 * different MTYPEs 2589 * - Need to allow room for fragmentation, guard pages etc. 2590 * 2591 * This adds up to a rough guess of system memory x3. 2592 * Round up to power of two to maximize the available 2593 * VM size with the given page table size. 2594 */ 2595 si_meminfo(&si); 2596 phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit + 2597 (1 << 30) - 1) >> 30; 2598 vm_size = roundup_pow_of_two( 2599 min(max(phys_ram_gb * 3, min_vm_size), max_size)); 2600 } 2601 2602 adev->vm_manager.max_pfn = (uint64_t)vm_size << 18; 2603 2604 tmp = roundup_pow_of_two(adev->vm_manager.max_pfn); 2605 if (amdgpu_vm_block_size != -1) 2606 tmp >>= amdgpu_vm_block_size - 9; 2607 tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1; 2608 adev->vm_manager.num_level = min(max_level, (unsigned)tmp); 2609 switch (adev->vm_manager.num_level) { 2610 case 3: 2611 adev->vm_manager.root_level = AMDGPU_VM_PDB2; 2612 break; 2613 case 2: 2614 adev->vm_manager.root_level = AMDGPU_VM_PDB1; 2615 break; 2616 case 1: 2617 adev->vm_manager.root_level = AMDGPU_VM_PDB0; 2618 break; 2619 default: 2620 dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n"); 2621 } 2622 /* block size depends on vm size and hw setup*/ 2623 if (amdgpu_vm_block_size != -1) 2624 adev->vm_manager.block_size = 2625 min((unsigned)amdgpu_vm_block_size, max_bits 2626 - AMDGPU_GPU_PAGE_SHIFT 2627 - 9 * adev->vm_manager.num_level); 2628 else if (adev->vm_manager.num_level > 1) 2629 adev->vm_manager.block_size = 9; 2630 else 2631 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp); 2632 2633 if (amdgpu_vm_fragment_size == -1) 2634 adev->vm_manager.fragment_size = fragment_size_default; 2635 else 2636 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size; 2637 2638 DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n", 2639 vm_size, adev->vm_manager.num_level + 1, 2640 adev->vm_manager.block_size, 2641 adev->vm_manager.fragment_size); 2642 } 2643 2644 /** 2645 * amdgpu_vm_wait_idle - wait for the VM to become idle 2646 * 2647 * @vm: VM object to wait for 2648 * @timeout: timeout to wait for VM to become idle 2649 */ 2650 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout) 2651 { 2652 return dma_resv_wait_timeout_rcu(vm->root.base.bo->tbo.base.resv, 2653 true, true, timeout); 2654 } 2655 2656 /** 2657 * amdgpu_vm_init - initialize a vm instance 2658 * 2659 * @adev: amdgpu_device pointer 2660 * @vm: requested vm 2661 * @vm_context: Indicates if it GFX or Compute context 2662 * @pasid: Process address space identifier 2663 * 2664 * Init @vm fields. 2665 * 2666 * Returns: 2667 * 0 for success, error for failure. 2668 */ 2669 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, 2670 int vm_context, unsigned int pasid) 2671 { 2672 struct amdgpu_bo_param bp; 2673 struct amdgpu_bo *root; 2674 int r, i; 2675 2676 vm->va = RB_ROOT_CACHED; 2677 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) 2678 vm->reserved_vmid[i] = NULL; 2679 INIT_LIST_HEAD(&vm->evicted); 2680 INIT_LIST_HEAD(&vm->relocated); 2681 INIT_LIST_HEAD(&vm->moved); 2682 INIT_LIST_HEAD(&vm->idle); 2683 INIT_LIST_HEAD(&vm->invalidated); 2684 spin_lock_init(&vm->invalidated_lock); 2685 INIT_LIST_HEAD(&vm->freed); 2686 2687 /* create scheduler entity for page table updates */ 2688 r = drm_sched_entity_init(&vm->entity, adev->vm_manager.vm_pte_rqs, 2689 adev->vm_manager.vm_pte_num_rqs, NULL); 2690 if (r) 2691 return r; 2692 2693 vm->pte_support_ats = false; 2694 2695 if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) { 2696 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2697 AMDGPU_VM_USE_CPU_FOR_COMPUTE); 2698 2699 if (adev->asic_type == CHIP_RAVEN) 2700 vm->pte_support_ats = true; 2701 } else { 2702 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2703 AMDGPU_VM_USE_CPU_FOR_GFX); 2704 } 2705 DRM_DEBUG_DRIVER("VM update mode is %s\n", 2706 vm->use_cpu_for_update ? "CPU" : "SDMA"); 2707 WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)), 2708 "CPU update of VM recommended only for large BAR system\n"); 2709 2710 if (vm->use_cpu_for_update) 2711 vm->update_funcs = &amdgpu_vm_cpu_funcs; 2712 else 2713 vm->update_funcs = &amdgpu_vm_sdma_funcs; 2714 vm->last_update = NULL; 2715 2716 amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, &bp); 2717 if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) 2718 bp.flags &= ~AMDGPU_GEM_CREATE_SHADOW; 2719 r = amdgpu_bo_create(adev, &bp, &root); 2720 if (r) 2721 goto error_free_sched_entity; 2722 2723 r = amdgpu_bo_reserve(root, true); 2724 if (r) 2725 goto error_free_root; 2726 2727 r = dma_resv_reserve_shared(root->tbo.base.resv, 1); 2728 if (r) 2729 goto error_unreserve; 2730 2731 amdgpu_vm_bo_base_init(&vm->root.base, vm, root); 2732 2733 r = amdgpu_vm_clear_bo(adev, vm, root); 2734 if (r) 2735 goto error_unreserve; 2736 2737 amdgpu_bo_unreserve(vm->root.base.bo); 2738 2739 if (pasid) { 2740 unsigned long flags; 2741 2742 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2743 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1, 2744 GFP_ATOMIC); 2745 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2746 if (r < 0) 2747 goto error_free_root; 2748 2749 vm->pasid = pasid; 2750 } 2751 2752 INIT_KFIFO(vm->faults); 2753 2754 return 0; 2755 2756 error_unreserve: 2757 amdgpu_bo_unreserve(vm->root.base.bo); 2758 2759 error_free_root: 2760 amdgpu_bo_unref(&vm->root.base.bo->shadow); 2761 amdgpu_bo_unref(&vm->root.base.bo); 2762 vm->root.base.bo = NULL; 2763 2764 error_free_sched_entity: 2765 drm_sched_entity_destroy(&vm->entity); 2766 2767 return r; 2768 } 2769 2770 /** 2771 * amdgpu_vm_check_clean_reserved - check if a VM is clean 2772 * 2773 * @adev: amdgpu_device pointer 2774 * @vm: the VM to check 2775 * 2776 * check all entries of the root PD, if any subsequent PDs are allocated, 2777 * it means there are page table creating and filling, and is no a clean 2778 * VM 2779 * 2780 * Returns: 2781 * 0 if this VM is clean 2782 */ 2783 static int amdgpu_vm_check_clean_reserved(struct amdgpu_device *adev, 2784 struct amdgpu_vm *vm) 2785 { 2786 enum amdgpu_vm_level root = adev->vm_manager.root_level; 2787 unsigned int entries = amdgpu_vm_num_entries(adev, root); 2788 unsigned int i = 0; 2789 2790 if (!(vm->root.entries)) 2791 return 0; 2792 2793 for (i = 0; i < entries; i++) { 2794 if (vm->root.entries[i].base.bo) 2795 return -EINVAL; 2796 } 2797 2798 return 0; 2799 } 2800 2801 /** 2802 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM 2803 * 2804 * @adev: amdgpu_device pointer 2805 * @vm: requested vm 2806 * 2807 * This only works on GFX VMs that don't have any BOs added and no 2808 * page tables allocated yet. 2809 * 2810 * Changes the following VM parameters: 2811 * - use_cpu_for_update 2812 * - pte_supports_ats 2813 * - pasid (old PASID is released, because compute manages its own PASIDs) 2814 * 2815 * Reinitializes the page directory to reflect the changed ATS 2816 * setting. 2817 * 2818 * Returns: 2819 * 0 for success, -errno for errors. 2820 */ 2821 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned int pasid) 2822 { 2823 bool pte_support_ats = (adev->asic_type == CHIP_RAVEN); 2824 int r; 2825 2826 r = amdgpu_bo_reserve(vm->root.base.bo, true); 2827 if (r) 2828 return r; 2829 2830 /* Sanity checks */ 2831 r = amdgpu_vm_check_clean_reserved(adev, vm); 2832 if (r) 2833 goto unreserve_bo; 2834 2835 if (pasid) { 2836 unsigned long flags; 2837 2838 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2839 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1, 2840 GFP_ATOMIC); 2841 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2842 2843 if (r == -ENOSPC) 2844 goto unreserve_bo; 2845 r = 0; 2846 } 2847 2848 /* Check if PD needs to be reinitialized and do it before 2849 * changing any other state, in case it fails. 2850 */ 2851 if (pte_support_ats != vm->pte_support_ats) { 2852 vm->pte_support_ats = pte_support_ats; 2853 r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo); 2854 if (r) 2855 goto free_idr; 2856 } 2857 2858 /* Update VM state */ 2859 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2860 AMDGPU_VM_USE_CPU_FOR_COMPUTE); 2861 DRM_DEBUG_DRIVER("VM update mode is %s\n", 2862 vm->use_cpu_for_update ? "CPU" : "SDMA"); 2863 WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)), 2864 "CPU update of VM recommended only for large BAR system\n"); 2865 2866 if (vm->use_cpu_for_update) 2867 vm->update_funcs = &amdgpu_vm_cpu_funcs; 2868 else 2869 vm->update_funcs = &amdgpu_vm_sdma_funcs; 2870 dma_fence_put(vm->last_update); 2871 vm->last_update = NULL; 2872 2873 if (vm->pasid) { 2874 unsigned long flags; 2875 2876 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2877 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); 2878 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2879 2880 /* Free the original amdgpu allocated pasid 2881 * Will be replaced with kfd allocated pasid 2882 */ 2883 amdgpu_pasid_free(vm->pasid); 2884 vm->pasid = 0; 2885 } 2886 2887 /* Free the shadow bo for compute VM */ 2888 amdgpu_bo_unref(&vm->root.base.bo->shadow); 2889 2890 if (pasid) 2891 vm->pasid = pasid; 2892 2893 goto unreserve_bo; 2894 2895 free_idr: 2896 if (pasid) { 2897 unsigned long flags; 2898 2899 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2900 idr_remove(&adev->vm_manager.pasid_idr, pasid); 2901 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2902 } 2903 unreserve_bo: 2904 amdgpu_bo_unreserve(vm->root.base.bo); 2905 return r; 2906 } 2907 2908 /** 2909 * amdgpu_vm_release_compute - release a compute vm 2910 * @adev: amdgpu_device pointer 2911 * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute 2912 * 2913 * This is a correspondant of amdgpu_vm_make_compute. It decouples compute 2914 * pasid from vm. Compute should stop use of vm after this call. 2915 */ 2916 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) 2917 { 2918 if (vm->pasid) { 2919 unsigned long flags; 2920 2921 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2922 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); 2923 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2924 } 2925 vm->pasid = 0; 2926 } 2927 2928 /** 2929 * amdgpu_vm_fini - tear down a vm instance 2930 * 2931 * @adev: amdgpu_device pointer 2932 * @vm: requested vm 2933 * 2934 * Tear down @vm. 2935 * Unbind the VM and remove all bos from the vm bo list 2936 */ 2937 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 2938 { 2939 struct amdgpu_bo_va_mapping *mapping, *tmp; 2940 bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt; 2941 struct amdgpu_bo *root; 2942 int i, r; 2943 2944 amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm); 2945 2946 if (vm->pasid) { 2947 unsigned long flags; 2948 2949 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2950 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); 2951 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2952 } 2953 2954 drm_sched_entity_destroy(&vm->entity); 2955 2956 if (!RB_EMPTY_ROOT(&vm->va.rb_root)) { 2957 dev_err(adev->dev, "still active bo inside vm\n"); 2958 } 2959 rbtree_postorder_for_each_entry_safe(mapping, tmp, 2960 &vm->va.rb_root, rb) { 2961 /* Don't remove the mapping here, we don't want to trigger a 2962 * rebalance and the tree is about to be destroyed anyway. 2963 */ 2964 list_del(&mapping->list); 2965 kfree(mapping); 2966 } 2967 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { 2968 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) { 2969 amdgpu_vm_prt_fini(adev, vm); 2970 prt_fini_needed = false; 2971 } 2972 2973 list_del(&mapping->list); 2974 amdgpu_vm_free_mapping(adev, vm, mapping, NULL); 2975 } 2976 2977 root = amdgpu_bo_ref(vm->root.base.bo); 2978 r = amdgpu_bo_reserve(root, true); 2979 if (r) { 2980 dev_err(adev->dev, "Leaking page tables because BO reservation failed\n"); 2981 } else { 2982 amdgpu_vm_free_pts(adev, vm, NULL); 2983 amdgpu_bo_unreserve(root); 2984 } 2985 amdgpu_bo_unref(&root); 2986 WARN_ON(vm->root.base.bo); 2987 dma_fence_put(vm->last_update); 2988 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) 2989 amdgpu_vmid_free_reserved(adev, vm, i); 2990 } 2991 2992 /** 2993 * amdgpu_vm_manager_init - init the VM manager 2994 * 2995 * @adev: amdgpu_device pointer 2996 * 2997 * Initialize the VM manager structures 2998 */ 2999 void amdgpu_vm_manager_init(struct amdgpu_device *adev) 3000 { 3001 unsigned i; 3002 3003 amdgpu_vmid_mgr_init(adev); 3004 3005 adev->vm_manager.fence_context = 3006 dma_fence_context_alloc(AMDGPU_MAX_RINGS); 3007 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) 3008 adev->vm_manager.seqno[i] = 0; 3009 3010 spin_lock_init(&adev->vm_manager.prt_lock); 3011 atomic_set(&adev->vm_manager.num_prt_users, 0); 3012 3013 /* If not overridden by the user, by default, only in large BAR systems 3014 * Compute VM tables will be updated by CPU 3015 */ 3016 #ifdef CONFIG_X86_64 3017 if (amdgpu_vm_update_mode == -1) { 3018 if (amdgpu_gmc_vram_full_visible(&adev->gmc)) 3019 adev->vm_manager.vm_update_mode = 3020 AMDGPU_VM_USE_CPU_FOR_COMPUTE; 3021 else 3022 adev->vm_manager.vm_update_mode = 0; 3023 } else 3024 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode; 3025 #else 3026 adev->vm_manager.vm_update_mode = 0; 3027 #endif 3028 3029 idr_init(&adev->vm_manager.pasid_idr); 3030 spin_lock_init(&adev->vm_manager.pasid_lock); 3031 3032 adev->vm_manager.xgmi_map_counter = 0; 3033 mutex_init(&adev->vm_manager.lock_pstate); 3034 } 3035 3036 /** 3037 * amdgpu_vm_manager_fini - cleanup VM manager 3038 * 3039 * @adev: amdgpu_device pointer 3040 * 3041 * Cleanup the VM manager and free resources. 3042 */ 3043 void amdgpu_vm_manager_fini(struct amdgpu_device *adev) 3044 { 3045 WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr)); 3046 idr_destroy(&adev->vm_manager.pasid_idr); 3047 3048 amdgpu_vmid_mgr_fini(adev); 3049 } 3050 3051 /** 3052 * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs. 3053 * 3054 * @dev: drm device pointer 3055 * @data: drm_amdgpu_vm 3056 * @filp: drm file pointer 3057 * 3058 * Returns: 3059 * 0 for success, -errno for errors. 3060 */ 3061 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 3062 { 3063 union drm_amdgpu_vm *args = data; 3064 struct amdgpu_device *adev = dev->dev_private; 3065 struct amdgpu_fpriv *fpriv = filp->driver_priv; 3066 int r; 3067 3068 switch (args->in.op) { 3069 case AMDGPU_VM_OP_RESERVE_VMID: 3070 /* current, we only have requirement to reserve vmid from gfxhub */ 3071 r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0); 3072 if (r) 3073 return r; 3074 break; 3075 case AMDGPU_VM_OP_UNRESERVE_VMID: 3076 amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0); 3077 break; 3078 default: 3079 return -EINVAL; 3080 } 3081 3082 return 0; 3083 } 3084 3085 /** 3086 * amdgpu_vm_get_task_info - Extracts task info for a PASID. 3087 * 3088 * @adev: drm device pointer 3089 * @pasid: PASID identifier for VM 3090 * @task_info: task_info to fill. 3091 */ 3092 void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid, 3093 struct amdgpu_task_info *task_info) 3094 { 3095 struct amdgpu_vm *vm; 3096 unsigned long flags; 3097 3098 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 3099 3100 vm = idr_find(&adev->vm_manager.pasid_idr, pasid); 3101 if (vm) 3102 *task_info = vm->task_info; 3103 3104 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 3105 } 3106 3107 /** 3108 * amdgpu_vm_set_task_info - Sets VMs task info. 3109 * 3110 * @vm: vm for which to set the info 3111 */ 3112 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm) 3113 { 3114 if (!vm->task_info.pid) { 3115 vm->task_info.pid = current->pid; 3116 get_task_comm(vm->task_info.task_name, current); 3117 3118 if (current->group_leader->mm == current->mm) { 3119 vm->task_info.tgid = current->group_leader->pid; 3120 get_task_comm(vm->task_info.process_name, current->group_leader); 3121 } 3122 } 3123 } 3124