1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2020-2021 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include <linux/types.h> 25 #include <linux/sched/task.h> 26 #include "amdgpu_sync.h" 27 #include "amdgpu_object.h" 28 #include "amdgpu_vm.h" 29 #include "amdgpu_mn.h" 30 #include "amdgpu.h" 31 #include "amdgpu_xgmi.h" 32 #include "kfd_priv.h" 33 #include "kfd_svm.h" 34 #include "kfd_migrate.h" 35 36 #define AMDGPU_SVM_RANGE_RESTORE_DELAY_MS 1 37 38 /* Long enough to ensure no retry fault comes after svm range is restored and 39 * page table is updated. 40 */ 41 #define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING 2000 42 43 static void svm_range_evict_svm_bo_worker(struct work_struct *work); 44 static bool 45 svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni, 46 const struct mmu_notifier_range *range, 47 unsigned long cur_seq); 48 49 static const struct mmu_interval_notifier_ops svm_range_mn_ops = { 50 .invalidate = svm_range_cpu_invalidate_pagetables, 51 }; 52 53 /** 54 * svm_range_unlink - unlink svm_range from lists and interval tree 55 * @prange: svm range structure to be removed 56 * 57 * Remove the svm_range from the svms and svm_bo lists and the svms 58 * interval tree. 59 * 60 * Context: The caller must hold svms->lock 61 */ 62 static void svm_range_unlink(struct svm_range *prange) 63 { 64 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, 65 prange, prange->start, prange->last); 66 67 if (prange->svm_bo) { 68 spin_lock(&prange->svm_bo->list_lock); 69 list_del(&prange->svm_bo_list); 70 spin_unlock(&prange->svm_bo->list_lock); 71 } 72 73 list_del(&prange->list); 74 if (prange->it_node.start != 0 && prange->it_node.last != 0) 75 interval_tree_remove(&prange->it_node, &prange->svms->objects); 76 } 77 78 static void 79 svm_range_add_notifier_locked(struct mm_struct *mm, struct svm_range *prange) 80 { 81 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, 82 prange, prange->start, prange->last); 83 84 mmu_interval_notifier_insert_locked(&prange->notifier, mm, 85 prange->start << PAGE_SHIFT, 86 prange->npages << PAGE_SHIFT, 87 &svm_range_mn_ops); 88 } 89 90 /** 91 * svm_range_add_to_svms - add svm range to svms 92 * @prange: svm range structure to be added 93 * 94 * Add the svm range to svms interval tree and link list 95 * 96 * Context: The caller must hold svms->lock 97 */ 98 static void svm_range_add_to_svms(struct svm_range *prange) 99 { 100 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, 101 prange, prange->start, prange->last); 102 103 list_add_tail(&prange->list, &prange->svms->list); 104 prange->it_node.start = prange->start; 105 prange->it_node.last = prange->last; 106 interval_tree_insert(&prange->it_node, &prange->svms->objects); 107 } 108 109 static void svm_range_remove_notifier(struct svm_range *prange) 110 { 111 pr_debug("remove notifier svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", 112 prange->svms, prange, 113 prange->notifier.interval_tree.start >> PAGE_SHIFT, 114 prange->notifier.interval_tree.last >> PAGE_SHIFT); 115 116 if (prange->notifier.interval_tree.start != 0 && 117 prange->notifier.interval_tree.last != 0) 118 mmu_interval_notifier_remove(&prange->notifier); 119 } 120 121 static bool 122 svm_is_valid_dma_mapping_addr(struct device *dev, dma_addr_t dma_addr) 123 { 124 return dma_addr && !dma_mapping_error(dev, dma_addr) && 125 !(dma_addr & SVM_RANGE_VRAM_DOMAIN); 126 } 127 128 static int 129 svm_range_dma_map_dev(struct amdgpu_device *adev, struct svm_range *prange, 130 unsigned long offset, unsigned long npages, 131 unsigned long *hmm_pfns, uint32_t gpuidx) 132 { 133 enum dma_data_direction dir = DMA_BIDIRECTIONAL; 134 dma_addr_t *addr = prange->dma_addr[gpuidx]; 135 struct device *dev = adev->dev; 136 struct page *page; 137 int i, r; 138 139 if (!addr) { 140 addr = kvmalloc_array(prange->npages, sizeof(*addr), 141 GFP_KERNEL | __GFP_ZERO); 142 if (!addr) 143 return -ENOMEM; 144 prange->dma_addr[gpuidx] = addr; 145 } 146 147 addr += offset; 148 for (i = 0; i < npages; i++) { 149 if (svm_is_valid_dma_mapping_addr(dev, addr[i])) 150 dma_unmap_page(dev, addr[i], PAGE_SIZE, dir); 151 152 page = hmm_pfn_to_page(hmm_pfns[i]); 153 if (is_zone_device_page(page)) { 154 struct amdgpu_device *bo_adev = 155 amdgpu_ttm_adev(prange->svm_bo->bo->tbo.bdev); 156 157 addr[i] = (hmm_pfns[i] << PAGE_SHIFT) + 158 bo_adev->vm_manager.vram_base_offset - 159 bo_adev->kfd.dev->pgmap.range.start; 160 addr[i] |= SVM_RANGE_VRAM_DOMAIN; 161 pr_debug("vram address detected: 0x%llx\n", addr[i]); 162 continue; 163 } 164 addr[i] = dma_map_page(dev, page, 0, PAGE_SIZE, dir); 165 r = dma_mapping_error(dev, addr[i]); 166 if (r) { 167 pr_debug("failed %d dma_map_page\n", r); 168 return r; 169 } 170 pr_debug("dma mapping 0x%llx for page addr 0x%lx\n", 171 addr[i] >> PAGE_SHIFT, page_to_pfn(page)); 172 } 173 return 0; 174 } 175 176 static int 177 svm_range_dma_map(struct svm_range *prange, unsigned long *bitmap, 178 unsigned long offset, unsigned long npages, 179 unsigned long *hmm_pfns) 180 { 181 struct kfd_process *p; 182 uint32_t gpuidx; 183 int r; 184 185 p = container_of(prange->svms, struct kfd_process, svms); 186 187 for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) { 188 struct kfd_process_device *pdd; 189 struct amdgpu_device *adev; 190 191 pr_debug("mapping to gpu idx 0x%x\n", gpuidx); 192 pdd = kfd_process_device_from_gpuidx(p, gpuidx); 193 if (!pdd) { 194 pr_debug("failed to find device idx %d\n", gpuidx); 195 return -EINVAL; 196 } 197 adev = (struct amdgpu_device *)pdd->dev->kgd; 198 199 r = svm_range_dma_map_dev(adev, prange, offset, npages, 200 hmm_pfns, gpuidx); 201 if (r) 202 break; 203 } 204 205 return r; 206 } 207 208 void svm_range_dma_unmap(struct device *dev, dma_addr_t *dma_addr, 209 unsigned long offset, unsigned long npages) 210 { 211 enum dma_data_direction dir = DMA_BIDIRECTIONAL; 212 int i; 213 214 if (!dma_addr) 215 return; 216 217 for (i = offset; i < offset + npages; i++) { 218 if (!svm_is_valid_dma_mapping_addr(dev, dma_addr[i])) 219 continue; 220 pr_debug("dma unmapping 0x%llx\n", dma_addr[i] >> PAGE_SHIFT); 221 dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir); 222 dma_addr[i] = 0; 223 } 224 } 225 226 void svm_range_free_dma_mappings(struct svm_range *prange) 227 { 228 struct kfd_process_device *pdd; 229 dma_addr_t *dma_addr; 230 struct device *dev; 231 struct kfd_process *p; 232 uint32_t gpuidx; 233 234 p = container_of(prange->svms, struct kfd_process, svms); 235 236 for (gpuidx = 0; gpuidx < MAX_GPU_INSTANCE; gpuidx++) { 237 dma_addr = prange->dma_addr[gpuidx]; 238 if (!dma_addr) 239 continue; 240 241 pdd = kfd_process_device_from_gpuidx(p, gpuidx); 242 if (!pdd) { 243 pr_debug("failed to find device idx %d\n", gpuidx); 244 continue; 245 } 246 dev = &pdd->dev->pdev->dev; 247 svm_range_dma_unmap(dev, dma_addr, 0, prange->npages); 248 kvfree(dma_addr); 249 prange->dma_addr[gpuidx] = NULL; 250 } 251 } 252 253 static void svm_range_free(struct svm_range *prange) 254 { 255 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, prange, 256 prange->start, prange->last); 257 258 svm_range_vram_node_free(prange); 259 svm_range_free_dma_mappings(prange); 260 mutex_destroy(&prange->lock); 261 mutex_destroy(&prange->migrate_mutex); 262 kfree(prange); 263 } 264 265 static void 266 svm_range_set_default_attributes(int32_t *location, int32_t *prefetch_loc, 267 uint8_t *granularity, uint32_t *flags) 268 { 269 *location = KFD_IOCTL_SVM_LOCATION_UNDEFINED; 270 *prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED; 271 *granularity = 9; 272 *flags = 273 KFD_IOCTL_SVM_FLAG_HOST_ACCESS | KFD_IOCTL_SVM_FLAG_COHERENT; 274 } 275 276 static struct 277 svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start, 278 uint64_t last) 279 { 280 uint64_t size = last - start + 1; 281 struct svm_range *prange; 282 struct kfd_process *p; 283 284 prange = kzalloc(sizeof(*prange), GFP_KERNEL); 285 if (!prange) 286 return NULL; 287 prange->npages = size; 288 prange->svms = svms; 289 prange->start = start; 290 prange->last = last; 291 INIT_LIST_HEAD(&prange->list); 292 INIT_LIST_HEAD(&prange->update_list); 293 INIT_LIST_HEAD(&prange->remove_list); 294 INIT_LIST_HEAD(&prange->insert_list); 295 INIT_LIST_HEAD(&prange->svm_bo_list); 296 INIT_LIST_HEAD(&prange->deferred_list); 297 INIT_LIST_HEAD(&prange->child_list); 298 atomic_set(&prange->invalid, 0); 299 prange->validate_timestamp = 0; 300 mutex_init(&prange->migrate_mutex); 301 mutex_init(&prange->lock); 302 303 p = container_of(svms, struct kfd_process, svms); 304 if (p->xnack_enabled) 305 bitmap_copy(prange->bitmap_access, svms->bitmap_supported, 306 MAX_GPU_INSTANCE); 307 308 svm_range_set_default_attributes(&prange->preferred_loc, 309 &prange->prefetch_loc, 310 &prange->granularity, &prange->flags); 311 312 pr_debug("svms 0x%p [0x%llx 0x%llx]\n", svms, start, last); 313 314 return prange; 315 } 316 317 static bool svm_bo_ref_unless_zero(struct svm_range_bo *svm_bo) 318 { 319 if (!svm_bo || !kref_get_unless_zero(&svm_bo->kref)) 320 return false; 321 322 return true; 323 } 324 325 static void svm_range_bo_release(struct kref *kref) 326 { 327 struct svm_range_bo *svm_bo; 328 329 svm_bo = container_of(kref, struct svm_range_bo, kref); 330 spin_lock(&svm_bo->list_lock); 331 while (!list_empty(&svm_bo->range_list)) { 332 struct svm_range *prange = 333 list_first_entry(&svm_bo->range_list, 334 struct svm_range, svm_bo_list); 335 /* list_del_init tells a concurrent svm_range_vram_node_new when 336 * it's safe to reuse the svm_bo pointer and svm_bo_list head. 337 */ 338 list_del_init(&prange->svm_bo_list); 339 spin_unlock(&svm_bo->list_lock); 340 341 pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, 342 prange->start, prange->last); 343 mutex_lock(&prange->lock); 344 prange->svm_bo = NULL; 345 mutex_unlock(&prange->lock); 346 347 spin_lock(&svm_bo->list_lock); 348 } 349 spin_unlock(&svm_bo->list_lock); 350 if (!dma_fence_is_signaled(&svm_bo->eviction_fence->base)) { 351 /* We're not in the eviction worker. 352 * Signal the fence and synchronize with any 353 * pending eviction work. 354 */ 355 dma_fence_signal(&svm_bo->eviction_fence->base); 356 cancel_work_sync(&svm_bo->eviction_work); 357 } 358 dma_fence_put(&svm_bo->eviction_fence->base); 359 amdgpu_bo_unref(&svm_bo->bo); 360 kfree(svm_bo); 361 } 362 363 void svm_range_bo_unref(struct svm_range_bo *svm_bo) 364 { 365 if (!svm_bo) 366 return; 367 368 kref_put(&svm_bo->kref, svm_range_bo_release); 369 } 370 371 static bool 372 svm_range_validate_svm_bo(struct amdgpu_device *adev, struct svm_range *prange) 373 { 374 struct amdgpu_device *bo_adev; 375 376 mutex_lock(&prange->lock); 377 if (!prange->svm_bo) { 378 mutex_unlock(&prange->lock); 379 return false; 380 } 381 if (prange->ttm_res) { 382 /* We still have a reference, all is well */ 383 mutex_unlock(&prange->lock); 384 return true; 385 } 386 if (svm_bo_ref_unless_zero(prange->svm_bo)) { 387 /* 388 * Migrate from GPU to GPU, remove range from source bo_adev 389 * svm_bo range list, and return false to allocate svm_bo from 390 * destination adev. 391 */ 392 bo_adev = amdgpu_ttm_adev(prange->svm_bo->bo->tbo.bdev); 393 if (bo_adev != adev) { 394 mutex_unlock(&prange->lock); 395 396 spin_lock(&prange->svm_bo->list_lock); 397 list_del_init(&prange->svm_bo_list); 398 spin_unlock(&prange->svm_bo->list_lock); 399 400 svm_range_bo_unref(prange->svm_bo); 401 return false; 402 } 403 if (READ_ONCE(prange->svm_bo->evicting)) { 404 struct dma_fence *f; 405 struct svm_range_bo *svm_bo; 406 /* The BO is getting evicted, 407 * we need to get a new one 408 */ 409 mutex_unlock(&prange->lock); 410 svm_bo = prange->svm_bo; 411 f = dma_fence_get(&svm_bo->eviction_fence->base); 412 svm_range_bo_unref(prange->svm_bo); 413 /* wait for the fence to avoid long spin-loop 414 * at list_empty_careful 415 */ 416 dma_fence_wait(f, false); 417 dma_fence_put(f); 418 } else { 419 /* The BO was still around and we got 420 * a new reference to it 421 */ 422 mutex_unlock(&prange->lock); 423 pr_debug("reuse old bo svms 0x%p [0x%lx 0x%lx]\n", 424 prange->svms, prange->start, prange->last); 425 426 prange->ttm_res = prange->svm_bo->bo->tbo.resource; 427 return true; 428 } 429 430 } else { 431 mutex_unlock(&prange->lock); 432 } 433 434 /* We need a new svm_bo. Spin-loop to wait for concurrent 435 * svm_range_bo_release to finish removing this range from 436 * its range list. After this, it is safe to reuse the 437 * svm_bo pointer and svm_bo_list head. 438 */ 439 while (!list_empty_careful(&prange->svm_bo_list)) 440 ; 441 442 return false; 443 } 444 445 static struct svm_range_bo *svm_range_bo_new(void) 446 { 447 struct svm_range_bo *svm_bo; 448 449 svm_bo = kzalloc(sizeof(*svm_bo), GFP_KERNEL); 450 if (!svm_bo) 451 return NULL; 452 453 kref_init(&svm_bo->kref); 454 INIT_LIST_HEAD(&svm_bo->range_list); 455 spin_lock_init(&svm_bo->list_lock); 456 457 return svm_bo; 458 } 459 460 int 461 svm_range_vram_node_new(struct amdgpu_device *adev, struct svm_range *prange, 462 bool clear) 463 { 464 struct amdgpu_bo_param bp; 465 struct svm_range_bo *svm_bo; 466 struct amdgpu_bo_user *ubo; 467 struct amdgpu_bo *bo; 468 struct kfd_process *p; 469 struct mm_struct *mm; 470 int r; 471 472 p = container_of(prange->svms, struct kfd_process, svms); 473 pr_debug("pasid: %x svms 0x%p [0x%lx 0x%lx]\n", p->pasid, prange->svms, 474 prange->start, prange->last); 475 476 if (svm_range_validate_svm_bo(adev, prange)) 477 return 0; 478 479 svm_bo = svm_range_bo_new(); 480 if (!svm_bo) { 481 pr_debug("failed to alloc svm bo\n"); 482 return -ENOMEM; 483 } 484 mm = get_task_mm(p->lead_thread); 485 if (!mm) { 486 pr_debug("failed to get mm\n"); 487 kfree(svm_bo); 488 return -ESRCH; 489 } 490 svm_bo->svms = prange->svms; 491 svm_bo->eviction_fence = 492 amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1), 493 mm, 494 svm_bo); 495 mmput(mm); 496 INIT_WORK(&svm_bo->eviction_work, svm_range_evict_svm_bo_worker); 497 svm_bo->evicting = 0; 498 memset(&bp, 0, sizeof(bp)); 499 bp.size = prange->npages * PAGE_SIZE; 500 bp.byte_align = PAGE_SIZE; 501 bp.domain = AMDGPU_GEM_DOMAIN_VRAM; 502 bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS; 503 bp.flags |= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0; 504 bp.flags |= AMDGPU_AMDKFD_CREATE_SVM_BO; 505 bp.type = ttm_bo_type_device; 506 bp.resv = NULL; 507 508 r = amdgpu_bo_create_user(adev, &bp, &ubo); 509 if (r) { 510 pr_debug("failed %d to create bo\n", r); 511 goto create_bo_failed; 512 } 513 bo = &ubo->bo; 514 r = amdgpu_bo_reserve(bo, true); 515 if (r) { 516 pr_debug("failed %d to reserve bo\n", r); 517 goto reserve_bo_failed; 518 } 519 520 r = dma_resv_reserve_shared(bo->tbo.base.resv, 1); 521 if (r) { 522 pr_debug("failed %d to reserve bo\n", r); 523 amdgpu_bo_unreserve(bo); 524 goto reserve_bo_failed; 525 } 526 amdgpu_bo_fence(bo, &svm_bo->eviction_fence->base, true); 527 528 amdgpu_bo_unreserve(bo); 529 530 svm_bo->bo = bo; 531 prange->svm_bo = svm_bo; 532 prange->ttm_res = bo->tbo.resource; 533 prange->offset = 0; 534 535 spin_lock(&svm_bo->list_lock); 536 list_add(&prange->svm_bo_list, &svm_bo->range_list); 537 spin_unlock(&svm_bo->list_lock); 538 539 return 0; 540 541 reserve_bo_failed: 542 amdgpu_bo_unref(&bo); 543 create_bo_failed: 544 dma_fence_put(&svm_bo->eviction_fence->base); 545 kfree(svm_bo); 546 prange->ttm_res = NULL; 547 548 return r; 549 } 550 551 void svm_range_vram_node_free(struct svm_range *prange) 552 { 553 svm_range_bo_unref(prange->svm_bo); 554 prange->ttm_res = NULL; 555 } 556 557 struct amdgpu_device * 558 svm_range_get_adev_by_id(struct svm_range *prange, uint32_t gpu_id) 559 { 560 struct kfd_process_device *pdd; 561 struct kfd_process *p; 562 int32_t gpu_idx; 563 564 p = container_of(prange->svms, struct kfd_process, svms); 565 566 gpu_idx = kfd_process_gpuidx_from_gpuid(p, gpu_id); 567 if (gpu_idx < 0) { 568 pr_debug("failed to get device by id 0x%x\n", gpu_id); 569 return NULL; 570 } 571 pdd = kfd_process_device_from_gpuidx(p, gpu_idx); 572 if (!pdd) { 573 pr_debug("failed to get device by idx 0x%x\n", gpu_idx); 574 return NULL; 575 } 576 577 return (struct amdgpu_device *)pdd->dev->kgd; 578 } 579 580 struct kfd_process_device * 581 svm_range_get_pdd_by_adev(struct svm_range *prange, struct amdgpu_device *adev) 582 { 583 struct kfd_process *p; 584 int32_t gpu_idx, gpuid; 585 int r; 586 587 p = container_of(prange->svms, struct kfd_process, svms); 588 589 r = kfd_process_gpuid_from_kgd(p, adev, &gpuid, &gpu_idx); 590 if (r) { 591 pr_debug("failed to get device id by adev %p\n", adev); 592 return NULL; 593 } 594 595 return kfd_process_device_from_gpuidx(p, gpu_idx); 596 } 597 598 static int svm_range_bo_validate(void *param, struct amdgpu_bo *bo) 599 { 600 struct ttm_operation_ctx ctx = { false, false }; 601 602 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM); 603 604 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 605 } 606 607 static int 608 svm_range_check_attr(struct kfd_process *p, 609 uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs) 610 { 611 uint32_t i; 612 613 for (i = 0; i < nattr; i++) { 614 uint32_t val = attrs[i].value; 615 int gpuidx = MAX_GPU_INSTANCE; 616 617 switch (attrs[i].type) { 618 case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: 619 if (val != KFD_IOCTL_SVM_LOCATION_SYSMEM && 620 val != KFD_IOCTL_SVM_LOCATION_UNDEFINED) 621 gpuidx = kfd_process_gpuidx_from_gpuid(p, val); 622 break; 623 case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: 624 if (val != KFD_IOCTL_SVM_LOCATION_SYSMEM) 625 gpuidx = kfd_process_gpuidx_from_gpuid(p, val); 626 break; 627 case KFD_IOCTL_SVM_ATTR_ACCESS: 628 case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE: 629 case KFD_IOCTL_SVM_ATTR_NO_ACCESS: 630 gpuidx = kfd_process_gpuidx_from_gpuid(p, val); 631 break; 632 case KFD_IOCTL_SVM_ATTR_SET_FLAGS: 633 break; 634 case KFD_IOCTL_SVM_ATTR_CLR_FLAGS: 635 break; 636 case KFD_IOCTL_SVM_ATTR_GRANULARITY: 637 break; 638 default: 639 pr_debug("unknown attr type 0x%x\n", attrs[i].type); 640 return -EINVAL; 641 } 642 643 if (gpuidx < 0) { 644 pr_debug("no GPU 0x%x found\n", val); 645 return -EINVAL; 646 } else if (gpuidx < MAX_GPU_INSTANCE && 647 !test_bit(gpuidx, p->svms.bitmap_supported)) { 648 pr_debug("GPU 0x%x not supported\n", val); 649 return -EINVAL; 650 } 651 } 652 653 return 0; 654 } 655 656 static void 657 svm_range_apply_attrs(struct kfd_process *p, struct svm_range *prange, 658 uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs) 659 { 660 uint32_t i; 661 int gpuidx; 662 663 for (i = 0; i < nattr; i++) { 664 switch (attrs[i].type) { 665 case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: 666 prange->preferred_loc = attrs[i].value; 667 break; 668 case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: 669 prange->prefetch_loc = attrs[i].value; 670 break; 671 case KFD_IOCTL_SVM_ATTR_ACCESS: 672 case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE: 673 case KFD_IOCTL_SVM_ATTR_NO_ACCESS: 674 gpuidx = kfd_process_gpuidx_from_gpuid(p, 675 attrs[i].value); 676 if (attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS) { 677 bitmap_clear(prange->bitmap_access, gpuidx, 1); 678 bitmap_clear(prange->bitmap_aip, gpuidx, 1); 679 } else if (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS) { 680 bitmap_set(prange->bitmap_access, gpuidx, 1); 681 bitmap_clear(prange->bitmap_aip, gpuidx, 1); 682 } else { 683 bitmap_clear(prange->bitmap_access, gpuidx, 1); 684 bitmap_set(prange->bitmap_aip, gpuidx, 1); 685 } 686 break; 687 case KFD_IOCTL_SVM_ATTR_SET_FLAGS: 688 prange->flags |= attrs[i].value; 689 break; 690 case KFD_IOCTL_SVM_ATTR_CLR_FLAGS: 691 prange->flags &= ~attrs[i].value; 692 break; 693 case KFD_IOCTL_SVM_ATTR_GRANULARITY: 694 prange->granularity = attrs[i].value; 695 break; 696 default: 697 WARN_ONCE(1, "svm_range_check_attrs wasn't called?"); 698 } 699 } 700 } 701 702 /** 703 * svm_range_debug_dump - print all range information from svms 704 * @svms: svm range list header 705 * 706 * debug output svm range start, end, prefetch location from svms 707 * interval tree and link list 708 * 709 * Context: The caller must hold svms->lock 710 */ 711 static void svm_range_debug_dump(struct svm_range_list *svms) 712 { 713 struct interval_tree_node *node; 714 struct svm_range *prange; 715 716 pr_debug("dump svms 0x%p list\n", svms); 717 pr_debug("range\tstart\tpage\tend\t\tlocation\n"); 718 719 list_for_each_entry(prange, &svms->list, list) { 720 pr_debug("0x%p 0x%lx\t0x%llx\t0x%llx\t0x%x\n", 721 prange, prange->start, prange->npages, 722 prange->start + prange->npages - 1, 723 prange->actual_loc); 724 } 725 726 pr_debug("dump svms 0x%p interval tree\n", svms); 727 pr_debug("range\tstart\tpage\tend\t\tlocation\n"); 728 node = interval_tree_iter_first(&svms->objects, 0, ~0ULL); 729 while (node) { 730 prange = container_of(node, struct svm_range, it_node); 731 pr_debug("0x%p 0x%lx\t0x%llx\t0x%llx\t0x%x\n", 732 prange, prange->start, prange->npages, 733 prange->start + prange->npages - 1, 734 prange->actual_loc); 735 node = interval_tree_iter_next(node, 0, ~0ULL); 736 } 737 } 738 739 static bool 740 svm_range_is_same_attrs(struct svm_range *old, struct svm_range *new) 741 { 742 return (old->prefetch_loc == new->prefetch_loc && 743 old->flags == new->flags && 744 old->granularity == new->granularity); 745 } 746 747 static int 748 svm_range_split_array(void *ppnew, void *ppold, size_t size, 749 uint64_t old_start, uint64_t old_n, 750 uint64_t new_start, uint64_t new_n) 751 { 752 unsigned char *new, *old, *pold; 753 uint64_t d; 754 755 if (!ppold) 756 return 0; 757 pold = *(unsigned char **)ppold; 758 if (!pold) 759 return 0; 760 761 new = kvmalloc_array(new_n, size, GFP_KERNEL); 762 if (!new) 763 return -ENOMEM; 764 765 d = (new_start - old_start) * size; 766 memcpy(new, pold + d, new_n * size); 767 768 old = kvmalloc_array(old_n, size, GFP_KERNEL); 769 if (!old) { 770 kvfree(new); 771 return -ENOMEM; 772 } 773 774 d = (new_start == old_start) ? new_n * size : 0; 775 memcpy(old, pold + d, old_n * size); 776 777 kvfree(pold); 778 *(void **)ppold = old; 779 *(void **)ppnew = new; 780 781 return 0; 782 } 783 784 static int 785 svm_range_split_pages(struct svm_range *new, struct svm_range *old, 786 uint64_t start, uint64_t last) 787 { 788 uint64_t npages = last - start + 1; 789 int i, r; 790 791 for (i = 0; i < MAX_GPU_INSTANCE; i++) { 792 r = svm_range_split_array(&new->dma_addr[i], &old->dma_addr[i], 793 sizeof(*old->dma_addr[i]), old->start, 794 npages, new->start, new->npages); 795 if (r) 796 return r; 797 } 798 799 return 0; 800 } 801 802 static int 803 svm_range_split_nodes(struct svm_range *new, struct svm_range *old, 804 uint64_t start, uint64_t last) 805 { 806 uint64_t npages = last - start + 1; 807 808 pr_debug("svms 0x%p new prange 0x%p start 0x%lx [0x%llx 0x%llx]\n", 809 new->svms, new, new->start, start, last); 810 811 if (new->start == old->start) { 812 new->offset = old->offset; 813 old->offset += new->npages; 814 } else { 815 new->offset = old->offset + npages; 816 } 817 818 new->svm_bo = svm_range_bo_ref(old->svm_bo); 819 new->ttm_res = old->ttm_res; 820 821 spin_lock(&new->svm_bo->list_lock); 822 list_add(&new->svm_bo_list, &new->svm_bo->range_list); 823 spin_unlock(&new->svm_bo->list_lock); 824 825 return 0; 826 } 827 828 /** 829 * svm_range_split_adjust - split range and adjust 830 * 831 * @new: new range 832 * @old: the old range 833 * @start: the old range adjust to start address in pages 834 * @last: the old range adjust to last address in pages 835 * 836 * Copy system memory dma_addr or vram ttm_res in old range to new 837 * range from new_start up to size new->npages, the remaining old range is from 838 * start to last 839 * 840 * Return: 841 * 0 - OK, -ENOMEM - out of memory 842 */ 843 static int 844 svm_range_split_adjust(struct svm_range *new, struct svm_range *old, 845 uint64_t start, uint64_t last) 846 { 847 int r; 848 849 pr_debug("svms 0x%p new 0x%lx old [0x%lx 0x%lx] => [0x%llx 0x%llx]\n", 850 new->svms, new->start, old->start, old->last, start, last); 851 852 if (new->start < old->start || 853 new->last > old->last) { 854 WARN_ONCE(1, "invalid new range start or last\n"); 855 return -EINVAL; 856 } 857 858 r = svm_range_split_pages(new, old, start, last); 859 if (r) 860 return r; 861 862 if (old->actual_loc && old->ttm_res) { 863 r = svm_range_split_nodes(new, old, start, last); 864 if (r) 865 return r; 866 } 867 868 old->npages = last - start + 1; 869 old->start = start; 870 old->last = last; 871 new->flags = old->flags; 872 new->preferred_loc = old->preferred_loc; 873 new->prefetch_loc = old->prefetch_loc; 874 new->actual_loc = old->actual_loc; 875 new->granularity = old->granularity; 876 bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE); 877 bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE); 878 879 return 0; 880 } 881 882 /** 883 * svm_range_split - split a range in 2 ranges 884 * 885 * @prange: the svm range to split 886 * @start: the remaining range start address in pages 887 * @last: the remaining range last address in pages 888 * @new: the result new range generated 889 * 890 * Two cases only: 891 * case 1: if start == prange->start 892 * prange ==> prange[start, last] 893 * new range [last + 1, prange->last] 894 * 895 * case 2: if last == prange->last 896 * prange ==> prange[start, last] 897 * new range [prange->start, start - 1] 898 * 899 * Return: 900 * 0 - OK, -ENOMEM - out of memory, -EINVAL - invalid start, last 901 */ 902 static int 903 svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last, 904 struct svm_range **new) 905 { 906 uint64_t old_start = prange->start; 907 uint64_t old_last = prange->last; 908 struct svm_range_list *svms; 909 int r = 0; 910 911 pr_debug("svms 0x%p [0x%llx 0x%llx] to [0x%llx 0x%llx]\n", prange->svms, 912 old_start, old_last, start, last); 913 914 if (old_start != start && old_last != last) 915 return -EINVAL; 916 if (start < old_start || last > old_last) 917 return -EINVAL; 918 919 svms = prange->svms; 920 if (old_start == start) 921 *new = svm_range_new(svms, last + 1, old_last); 922 else 923 *new = svm_range_new(svms, old_start, start - 1); 924 if (!*new) 925 return -ENOMEM; 926 927 r = svm_range_split_adjust(*new, prange, start, last); 928 if (r) { 929 pr_debug("failed %d split [0x%llx 0x%llx] to [0x%llx 0x%llx]\n", 930 r, old_start, old_last, start, last); 931 svm_range_free(*new); 932 *new = NULL; 933 } 934 935 return r; 936 } 937 938 static int 939 svm_range_split_tail(struct svm_range *prange, struct svm_range *new, 940 uint64_t new_last, struct list_head *insert_list) 941 { 942 struct svm_range *tail; 943 int r = svm_range_split(prange, prange->start, new_last, &tail); 944 945 if (!r) 946 list_add(&tail->insert_list, insert_list); 947 return r; 948 } 949 950 static int 951 svm_range_split_head(struct svm_range *prange, struct svm_range *new, 952 uint64_t new_start, struct list_head *insert_list) 953 { 954 struct svm_range *head; 955 int r = svm_range_split(prange, new_start, prange->last, &head); 956 957 if (!r) 958 list_add(&head->insert_list, insert_list); 959 return r; 960 } 961 962 static void 963 svm_range_add_child(struct svm_range *prange, struct mm_struct *mm, 964 struct svm_range *pchild, enum svm_work_list_ops op) 965 { 966 pr_debug("add child 0x%p [0x%lx 0x%lx] to prange 0x%p child list %d\n", 967 pchild, pchild->start, pchild->last, prange, op); 968 969 pchild->work_item.mm = mm; 970 pchild->work_item.op = op; 971 list_add_tail(&pchild->child_list, &prange->child_list); 972 } 973 974 /** 975 * svm_range_split_by_granularity - collect ranges within granularity boundary 976 * 977 * @p: the process with svms list 978 * @mm: mm structure 979 * @addr: the vm fault address in pages, to split the prange 980 * @parent: parent range if prange is from child list 981 * @prange: prange to split 982 * 983 * Trims @prange to be a single aligned block of prange->granularity if 984 * possible. The head and tail are added to the child_list in @parent. 985 * 986 * Context: caller must hold mmap_read_lock and prange->lock 987 * 988 * Return: 989 * 0 - OK, otherwise error code 990 */ 991 int 992 svm_range_split_by_granularity(struct kfd_process *p, struct mm_struct *mm, 993 unsigned long addr, struct svm_range *parent, 994 struct svm_range *prange) 995 { 996 struct svm_range *head, *tail; 997 unsigned long start, last, size; 998 int r; 999 1000 /* Align splited range start and size to granularity size, then a single 1001 * PTE will be used for whole range, this reduces the number of PTE 1002 * updated and the L1 TLB space used for translation. 1003 */ 1004 size = 1UL << prange->granularity; 1005 start = ALIGN_DOWN(addr, size); 1006 last = ALIGN(addr + 1, size) - 1; 1007 1008 pr_debug("svms 0x%p split [0x%lx 0x%lx] to [0x%lx 0x%lx] size 0x%lx\n", 1009 prange->svms, prange->start, prange->last, start, last, size); 1010 1011 if (start > prange->start) { 1012 r = svm_range_split(prange, start, prange->last, &head); 1013 if (r) 1014 return r; 1015 svm_range_add_child(parent, mm, head, SVM_OP_ADD_RANGE); 1016 } 1017 1018 if (last < prange->last) { 1019 r = svm_range_split(prange, prange->start, last, &tail); 1020 if (r) 1021 return r; 1022 svm_range_add_child(parent, mm, tail, SVM_OP_ADD_RANGE); 1023 } 1024 1025 /* xnack on, update mapping on GPUs with ACCESS_IN_PLACE */ 1026 if (p->xnack_enabled && prange->work_item.op == SVM_OP_ADD_RANGE) { 1027 prange->work_item.op = SVM_OP_ADD_RANGE_AND_MAP; 1028 pr_debug("change prange 0x%p [0x%lx 0x%lx] op %d\n", 1029 prange, prange->start, prange->last, 1030 SVM_OP_ADD_RANGE_AND_MAP); 1031 } 1032 return 0; 1033 } 1034 1035 static uint64_t 1036 svm_range_get_pte_flags(struct amdgpu_device *adev, struct svm_range *prange, 1037 int domain) 1038 { 1039 struct amdgpu_device *bo_adev; 1040 uint32_t flags = prange->flags; 1041 uint32_t mapping_flags = 0; 1042 uint64_t pte_flags; 1043 bool snoop = (domain != SVM_RANGE_VRAM_DOMAIN); 1044 bool coherent = flags & KFD_IOCTL_SVM_FLAG_COHERENT; 1045 1046 if (domain == SVM_RANGE_VRAM_DOMAIN) 1047 bo_adev = amdgpu_ttm_adev(prange->svm_bo->bo->tbo.bdev); 1048 1049 switch (adev->asic_type) { 1050 case CHIP_ARCTURUS: 1051 if (domain == SVM_RANGE_VRAM_DOMAIN) { 1052 if (bo_adev == adev) { 1053 mapping_flags |= coherent ? 1054 AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW; 1055 } else { 1056 mapping_flags |= coherent ? 1057 AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; 1058 if (amdgpu_xgmi_same_hive(adev, bo_adev)) 1059 snoop = true; 1060 } 1061 } else { 1062 mapping_flags |= coherent ? 1063 AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; 1064 } 1065 break; 1066 case CHIP_ALDEBARAN: 1067 if (domain == SVM_RANGE_VRAM_DOMAIN) { 1068 if (bo_adev == adev) { 1069 mapping_flags |= coherent ? 1070 AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW; 1071 if (adev->gmc.xgmi.connected_to_cpu) 1072 snoop = true; 1073 } else { 1074 mapping_flags |= coherent ? 1075 AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; 1076 if (amdgpu_xgmi_same_hive(adev, bo_adev)) 1077 snoop = true; 1078 } 1079 } else { 1080 mapping_flags |= coherent ? 1081 AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; 1082 } 1083 break; 1084 default: 1085 mapping_flags |= coherent ? 1086 AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; 1087 } 1088 1089 mapping_flags |= AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE; 1090 1091 if (flags & KFD_IOCTL_SVM_FLAG_GPU_RO) 1092 mapping_flags &= ~AMDGPU_VM_PAGE_WRITEABLE; 1093 if (flags & KFD_IOCTL_SVM_FLAG_GPU_EXEC) 1094 mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; 1095 1096 pte_flags = AMDGPU_PTE_VALID; 1097 pte_flags |= (domain == SVM_RANGE_VRAM_DOMAIN) ? 0 : AMDGPU_PTE_SYSTEM; 1098 pte_flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; 1099 1100 pte_flags |= amdgpu_gem_va_map_flags(adev, mapping_flags); 1101 return pte_flags; 1102 } 1103 1104 static int 1105 svm_range_unmap_from_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, 1106 uint64_t start, uint64_t last, 1107 struct dma_fence **fence) 1108 { 1109 uint64_t init_pte_value = 0; 1110 1111 pr_debug("[0x%llx 0x%llx]\n", start, last); 1112 1113 return amdgpu_vm_bo_update_mapping(adev, adev, vm, false, true, NULL, 1114 start, last, init_pte_value, 0, 1115 NULL, NULL, fence, NULL); 1116 } 1117 1118 static int 1119 svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start, 1120 unsigned long last) 1121 { 1122 DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE); 1123 struct kfd_process_device *pdd; 1124 struct dma_fence *fence = NULL; 1125 struct amdgpu_device *adev; 1126 struct kfd_process *p; 1127 uint32_t gpuidx; 1128 int r = 0; 1129 1130 bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip, 1131 MAX_GPU_INSTANCE); 1132 p = container_of(prange->svms, struct kfd_process, svms); 1133 1134 for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) { 1135 pr_debug("unmap from gpu idx 0x%x\n", gpuidx); 1136 pdd = kfd_process_device_from_gpuidx(p, gpuidx); 1137 if (!pdd) { 1138 pr_debug("failed to find device idx %d\n", gpuidx); 1139 return -EINVAL; 1140 } 1141 adev = (struct amdgpu_device *)pdd->dev->kgd; 1142 1143 r = svm_range_unmap_from_gpu(adev, drm_priv_to_vm(pdd->drm_priv), 1144 start, last, &fence); 1145 if (r) 1146 break; 1147 1148 if (fence) { 1149 r = dma_fence_wait(fence, false); 1150 dma_fence_put(fence); 1151 fence = NULL; 1152 if (r) 1153 break; 1154 } 1155 amdgpu_amdkfd_flush_gpu_tlb_pasid((struct kgd_dev *)adev, 1156 p->pasid, TLB_FLUSH_HEAVYWEIGHT); 1157 } 1158 1159 return r; 1160 } 1161 1162 static int 1163 svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm, 1164 struct svm_range *prange, unsigned long offset, 1165 unsigned long npages, bool readonly, dma_addr_t *dma_addr, 1166 struct amdgpu_device *bo_adev, struct dma_fence **fence) 1167 { 1168 struct amdgpu_bo_va bo_va; 1169 bool table_freed = false; 1170 uint64_t pte_flags; 1171 unsigned long last_start; 1172 int last_domain; 1173 int r = 0; 1174 int64_t i, j; 1175 1176 last_start = prange->start + offset; 1177 1178 pr_debug("svms 0x%p [0x%lx 0x%lx] readonly %d\n", prange->svms, 1179 last_start, last_start + npages - 1, readonly); 1180 1181 if (prange->svm_bo && prange->ttm_res) 1182 bo_va.is_xgmi = amdgpu_xgmi_same_hive(adev, bo_adev); 1183 1184 for (i = offset; i < offset + npages; i++) { 1185 last_domain = dma_addr[i] & SVM_RANGE_VRAM_DOMAIN; 1186 dma_addr[i] &= ~SVM_RANGE_VRAM_DOMAIN; 1187 1188 /* Collect all pages in the same address range and memory domain 1189 * that can be mapped with a single call to update mapping. 1190 */ 1191 if (i < offset + npages - 1 && 1192 last_domain == (dma_addr[i + 1] & SVM_RANGE_VRAM_DOMAIN)) 1193 continue; 1194 1195 pr_debug("Mapping range [0x%lx 0x%llx] on domain: %s\n", 1196 last_start, prange->start + i, last_domain ? "GPU" : "CPU"); 1197 1198 pte_flags = svm_range_get_pte_flags(adev, prange, last_domain); 1199 if (readonly) 1200 pte_flags &= ~AMDGPU_PTE_WRITEABLE; 1201 1202 pr_debug("svms 0x%p map [0x%lx 0x%llx] vram %d PTE 0x%llx\n", 1203 prange->svms, last_start, prange->start + i, 1204 (last_domain == SVM_RANGE_VRAM_DOMAIN) ? 1 : 0, 1205 pte_flags); 1206 1207 r = amdgpu_vm_bo_update_mapping(adev, bo_adev, vm, false, false, 1208 NULL, last_start, 1209 prange->start + i, pte_flags, 1210 last_start - prange->start, 1211 NULL, dma_addr, 1212 &vm->last_update, 1213 &table_freed); 1214 1215 for (j = last_start - prange->start; j <= i; j++) 1216 dma_addr[j] |= last_domain; 1217 1218 if (r) { 1219 pr_debug("failed %d to map to gpu 0x%lx\n", r, prange->start); 1220 goto out; 1221 } 1222 last_start = prange->start + i + 1; 1223 } 1224 1225 r = amdgpu_vm_update_pdes(adev, vm, false); 1226 if (r) { 1227 pr_debug("failed %d to update directories 0x%lx\n", r, 1228 prange->start); 1229 goto out; 1230 } 1231 1232 if (fence) 1233 *fence = dma_fence_get(vm->last_update); 1234 1235 if (table_freed) { 1236 struct kfd_process *p; 1237 1238 p = container_of(prange->svms, struct kfd_process, svms); 1239 amdgpu_amdkfd_flush_gpu_tlb_pasid((struct kgd_dev *)adev, 1240 p->pasid, TLB_FLUSH_LEGACY); 1241 } 1242 out: 1243 return r; 1244 } 1245 1246 static int 1247 svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset, 1248 unsigned long npages, bool readonly, 1249 unsigned long *bitmap, bool wait) 1250 { 1251 struct kfd_process_device *pdd; 1252 struct amdgpu_device *bo_adev; 1253 struct amdgpu_device *adev; 1254 struct kfd_process *p; 1255 struct dma_fence *fence = NULL; 1256 uint32_t gpuidx; 1257 int r = 0; 1258 1259 if (prange->svm_bo && prange->ttm_res) 1260 bo_adev = amdgpu_ttm_adev(prange->svm_bo->bo->tbo.bdev); 1261 else 1262 bo_adev = NULL; 1263 1264 p = container_of(prange->svms, struct kfd_process, svms); 1265 for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) { 1266 pr_debug("mapping to gpu idx 0x%x\n", gpuidx); 1267 pdd = kfd_process_device_from_gpuidx(p, gpuidx); 1268 if (!pdd) { 1269 pr_debug("failed to find device idx %d\n", gpuidx); 1270 return -EINVAL; 1271 } 1272 adev = (struct amdgpu_device *)pdd->dev->kgd; 1273 1274 pdd = kfd_bind_process_to_device(pdd->dev, p); 1275 if (IS_ERR(pdd)) 1276 return -EINVAL; 1277 1278 if (bo_adev && adev != bo_adev && 1279 !amdgpu_xgmi_same_hive(adev, bo_adev)) { 1280 pr_debug("cannot map to device idx %d\n", gpuidx); 1281 continue; 1282 } 1283 1284 r = svm_range_map_to_gpu(adev, drm_priv_to_vm(pdd->drm_priv), 1285 prange, offset, npages, readonly, 1286 prange->dma_addr[gpuidx], 1287 bo_adev, wait ? &fence : NULL); 1288 if (r) 1289 break; 1290 1291 if (fence) { 1292 r = dma_fence_wait(fence, false); 1293 dma_fence_put(fence); 1294 fence = NULL; 1295 if (r) { 1296 pr_debug("failed %d to dma fence wait\n", r); 1297 break; 1298 } 1299 } 1300 } 1301 1302 return r; 1303 } 1304 1305 struct svm_validate_context { 1306 struct kfd_process *process; 1307 struct svm_range *prange; 1308 bool intr; 1309 unsigned long bitmap[MAX_GPU_INSTANCE]; 1310 struct ttm_validate_buffer tv[MAX_GPU_INSTANCE]; 1311 struct list_head validate_list; 1312 struct ww_acquire_ctx ticket; 1313 }; 1314 1315 static int svm_range_reserve_bos(struct svm_validate_context *ctx) 1316 { 1317 struct kfd_process_device *pdd; 1318 struct amdgpu_device *adev; 1319 struct amdgpu_vm *vm; 1320 uint32_t gpuidx; 1321 int r; 1322 1323 INIT_LIST_HEAD(&ctx->validate_list); 1324 for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) { 1325 pdd = kfd_process_device_from_gpuidx(ctx->process, gpuidx); 1326 if (!pdd) { 1327 pr_debug("failed to find device idx %d\n", gpuidx); 1328 return -EINVAL; 1329 } 1330 adev = (struct amdgpu_device *)pdd->dev->kgd; 1331 vm = drm_priv_to_vm(pdd->drm_priv); 1332 1333 ctx->tv[gpuidx].bo = &vm->root.bo->tbo; 1334 ctx->tv[gpuidx].num_shared = 4; 1335 list_add(&ctx->tv[gpuidx].head, &ctx->validate_list); 1336 } 1337 1338 r = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->validate_list, 1339 ctx->intr, NULL); 1340 if (r) { 1341 pr_debug("failed %d to reserve bo\n", r); 1342 return r; 1343 } 1344 1345 for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) { 1346 pdd = kfd_process_device_from_gpuidx(ctx->process, gpuidx); 1347 if (!pdd) { 1348 pr_debug("failed to find device idx %d\n", gpuidx); 1349 r = -EINVAL; 1350 goto unreserve_out; 1351 } 1352 adev = (struct amdgpu_device *)pdd->dev->kgd; 1353 1354 r = amdgpu_vm_validate_pt_bos(adev, drm_priv_to_vm(pdd->drm_priv), 1355 svm_range_bo_validate, NULL); 1356 if (r) { 1357 pr_debug("failed %d validate pt bos\n", r); 1358 goto unreserve_out; 1359 } 1360 } 1361 1362 return 0; 1363 1364 unreserve_out: 1365 ttm_eu_backoff_reservation(&ctx->ticket, &ctx->validate_list); 1366 return r; 1367 } 1368 1369 static void svm_range_unreserve_bos(struct svm_validate_context *ctx) 1370 { 1371 ttm_eu_backoff_reservation(&ctx->ticket, &ctx->validate_list); 1372 } 1373 1374 static void *kfd_svm_page_owner(struct kfd_process *p, int32_t gpuidx) 1375 { 1376 struct kfd_process_device *pdd; 1377 struct amdgpu_device *adev; 1378 1379 pdd = kfd_process_device_from_gpuidx(p, gpuidx); 1380 adev = (struct amdgpu_device *)pdd->dev->kgd; 1381 1382 return SVM_ADEV_PGMAP_OWNER(adev); 1383 } 1384 1385 /* 1386 * Validation+GPU mapping with concurrent invalidation (MMU notifiers) 1387 * 1388 * To prevent concurrent destruction or change of range attributes, the 1389 * svm_read_lock must be held. The caller must not hold the svm_write_lock 1390 * because that would block concurrent evictions and lead to deadlocks. To 1391 * serialize concurrent migrations or validations of the same range, the 1392 * prange->migrate_mutex must be held. 1393 * 1394 * For VRAM ranges, the SVM BO must be allocated and valid (protected by its 1395 * eviction fence. 1396 * 1397 * The following sequence ensures race-free validation and GPU mapping: 1398 * 1399 * 1. Reserve page table (and SVM BO if range is in VRAM) 1400 * 2. hmm_range_fault to get page addresses (if system memory) 1401 * 3. DMA-map pages (if system memory) 1402 * 4-a. Take notifier lock 1403 * 4-b. Check that pages still valid (mmu_interval_read_retry) 1404 * 4-c. Check that the range was not split or otherwise invalidated 1405 * 4-d. Update GPU page table 1406 * 4.e. Release notifier lock 1407 * 5. Release page table (and SVM BO) reservation 1408 */ 1409 static int svm_range_validate_and_map(struct mm_struct *mm, 1410 struct svm_range *prange, 1411 int32_t gpuidx, bool intr, bool wait) 1412 { 1413 struct svm_validate_context ctx; 1414 unsigned long start, end, addr; 1415 struct kfd_process *p; 1416 void *owner; 1417 int32_t idx; 1418 int r = 0; 1419 1420 ctx.process = container_of(prange->svms, struct kfd_process, svms); 1421 ctx.prange = prange; 1422 ctx.intr = intr; 1423 1424 if (gpuidx < MAX_GPU_INSTANCE) { 1425 bitmap_zero(ctx.bitmap, MAX_GPU_INSTANCE); 1426 bitmap_set(ctx.bitmap, gpuidx, 1); 1427 } else if (ctx.process->xnack_enabled) { 1428 bitmap_copy(ctx.bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE); 1429 1430 /* If prefetch range to GPU, or GPU retry fault migrate range to 1431 * GPU, which has ACCESS attribute to the range, create mapping 1432 * on that GPU. 1433 */ 1434 if (prange->actual_loc) { 1435 gpuidx = kfd_process_gpuidx_from_gpuid(ctx.process, 1436 prange->actual_loc); 1437 if (gpuidx < 0) { 1438 WARN_ONCE(1, "failed get device by id 0x%x\n", 1439 prange->actual_loc); 1440 return -EINVAL; 1441 } 1442 if (test_bit(gpuidx, prange->bitmap_access)) 1443 bitmap_set(ctx.bitmap, gpuidx, 1); 1444 } 1445 } else { 1446 bitmap_or(ctx.bitmap, prange->bitmap_access, 1447 prange->bitmap_aip, MAX_GPU_INSTANCE); 1448 } 1449 1450 if (bitmap_empty(ctx.bitmap, MAX_GPU_INSTANCE)) 1451 return 0; 1452 1453 if (prange->actual_loc && !prange->ttm_res) { 1454 /* This should never happen. actual_loc gets set by 1455 * svm_migrate_ram_to_vram after allocating a BO. 1456 */ 1457 WARN(1, "VRAM BO missing during validation\n"); 1458 return -EINVAL; 1459 } 1460 1461 svm_range_reserve_bos(&ctx); 1462 1463 p = container_of(prange->svms, struct kfd_process, svms); 1464 owner = kfd_svm_page_owner(p, find_first_bit(ctx.bitmap, 1465 MAX_GPU_INSTANCE)); 1466 for_each_set_bit(idx, ctx.bitmap, MAX_GPU_INSTANCE) { 1467 if (kfd_svm_page_owner(p, idx) != owner) { 1468 owner = NULL; 1469 break; 1470 } 1471 } 1472 1473 start = prange->start << PAGE_SHIFT; 1474 end = (prange->last + 1) << PAGE_SHIFT; 1475 for (addr = start; addr < end && !r; ) { 1476 struct hmm_range *hmm_range; 1477 struct vm_area_struct *vma; 1478 unsigned long next; 1479 unsigned long offset; 1480 unsigned long npages; 1481 bool readonly; 1482 1483 vma = find_vma(mm, addr); 1484 if (!vma || addr < vma->vm_start) { 1485 r = -EFAULT; 1486 goto unreserve_out; 1487 } 1488 readonly = !(vma->vm_flags & VM_WRITE); 1489 1490 next = min(vma->vm_end, end); 1491 npages = (next - addr) >> PAGE_SHIFT; 1492 r = amdgpu_hmm_range_get_pages(&prange->notifier, mm, NULL, 1493 addr, npages, &hmm_range, 1494 readonly, true, owner); 1495 if (r) { 1496 pr_debug("failed %d to get svm range pages\n", r); 1497 goto unreserve_out; 1498 } 1499 1500 offset = (addr - start) >> PAGE_SHIFT; 1501 r = svm_range_dma_map(prange, ctx.bitmap, offset, npages, 1502 hmm_range->hmm_pfns); 1503 if (r) { 1504 pr_debug("failed %d to dma map range\n", r); 1505 goto unreserve_out; 1506 } 1507 1508 svm_range_lock(prange); 1509 if (amdgpu_hmm_range_get_pages_done(hmm_range)) { 1510 pr_debug("hmm update the range, need validate again\n"); 1511 r = -EAGAIN; 1512 goto unlock_out; 1513 } 1514 if (!list_empty(&prange->child_list)) { 1515 pr_debug("range split by unmap in parallel, validate again\n"); 1516 r = -EAGAIN; 1517 goto unlock_out; 1518 } 1519 1520 r = svm_range_map_to_gpus(prange, offset, npages, readonly, 1521 ctx.bitmap, wait); 1522 1523 unlock_out: 1524 svm_range_unlock(prange); 1525 1526 addr = next; 1527 } 1528 1529 if (addr == end) 1530 prange->validated_once = true; 1531 1532 unreserve_out: 1533 svm_range_unreserve_bos(&ctx); 1534 1535 if (!r) 1536 prange->validate_timestamp = ktime_to_us(ktime_get()); 1537 1538 return r; 1539 } 1540 1541 /** 1542 * svm_range_list_lock_and_flush_work - flush pending deferred work 1543 * 1544 * @svms: the svm range list 1545 * @mm: the mm structure 1546 * 1547 * Context: Returns with mmap write lock held, pending deferred work flushed 1548 * 1549 */ 1550 static void 1551 svm_range_list_lock_and_flush_work(struct svm_range_list *svms, 1552 struct mm_struct *mm) 1553 { 1554 retry_flush_work: 1555 flush_work(&svms->deferred_list_work); 1556 mmap_write_lock(mm); 1557 1558 if (list_empty(&svms->deferred_range_list)) 1559 return; 1560 mmap_write_unlock(mm); 1561 pr_debug("retry flush\n"); 1562 goto retry_flush_work; 1563 } 1564 1565 static void svm_range_restore_work(struct work_struct *work) 1566 { 1567 struct delayed_work *dwork = to_delayed_work(work); 1568 struct amdkfd_process_info *process_info; 1569 struct svm_range_list *svms; 1570 struct svm_range *prange; 1571 struct kfd_process *p; 1572 struct mm_struct *mm; 1573 int evicted_ranges; 1574 int invalid; 1575 int r; 1576 1577 svms = container_of(dwork, struct svm_range_list, restore_work); 1578 evicted_ranges = atomic_read(&svms->evicted_ranges); 1579 if (!evicted_ranges) 1580 return; 1581 1582 pr_debug("restore svm ranges\n"); 1583 1584 /* kfd_process_notifier_release destroys this worker thread. So during 1585 * the lifetime of this thread, kfd_process and mm will be valid. 1586 */ 1587 p = container_of(svms, struct kfd_process, svms); 1588 process_info = p->kgd_process_info; 1589 mm = p->mm; 1590 if (!mm) 1591 return; 1592 1593 mutex_lock(&process_info->lock); 1594 svm_range_list_lock_and_flush_work(svms, mm); 1595 mutex_lock(&svms->lock); 1596 1597 evicted_ranges = atomic_read(&svms->evicted_ranges); 1598 1599 list_for_each_entry(prange, &svms->list, list) { 1600 invalid = atomic_read(&prange->invalid); 1601 if (!invalid) 1602 continue; 1603 1604 pr_debug("restoring svms 0x%p prange 0x%p [0x%lx %lx] inv %d\n", 1605 prange->svms, prange, prange->start, prange->last, 1606 invalid); 1607 1608 /* 1609 * If range is migrating, wait for migration is done. 1610 */ 1611 mutex_lock(&prange->migrate_mutex); 1612 1613 r = svm_range_validate_and_map(mm, prange, MAX_GPU_INSTANCE, 1614 false, true); 1615 if (r) 1616 pr_debug("failed %d to map 0x%lx to gpus\n", r, 1617 prange->start); 1618 1619 mutex_unlock(&prange->migrate_mutex); 1620 if (r) 1621 goto out_reschedule; 1622 1623 if (atomic_cmpxchg(&prange->invalid, invalid, 0) != invalid) 1624 goto out_reschedule; 1625 } 1626 1627 if (atomic_cmpxchg(&svms->evicted_ranges, evicted_ranges, 0) != 1628 evicted_ranges) 1629 goto out_reschedule; 1630 1631 evicted_ranges = 0; 1632 1633 r = kgd2kfd_resume_mm(mm); 1634 if (r) { 1635 /* No recovery from this failure. Probably the CP is 1636 * hanging. No point trying again. 1637 */ 1638 pr_debug("failed %d to resume KFD\n", r); 1639 } 1640 1641 pr_debug("restore svm ranges successfully\n"); 1642 1643 out_reschedule: 1644 mutex_unlock(&svms->lock); 1645 mmap_write_unlock(mm); 1646 mutex_unlock(&process_info->lock); 1647 1648 /* If validation failed, reschedule another attempt */ 1649 if (evicted_ranges) { 1650 pr_debug("reschedule to restore svm range\n"); 1651 schedule_delayed_work(&svms->restore_work, 1652 msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS)); 1653 } 1654 } 1655 1656 /** 1657 * svm_range_evict - evict svm range 1658 * 1659 * Stop all queues of the process to ensure GPU doesn't access the memory, then 1660 * return to let CPU evict the buffer and proceed CPU pagetable update. 1661 * 1662 * Don't need use lock to sync cpu pagetable invalidation with GPU execution. 1663 * If invalidation happens while restore work is running, restore work will 1664 * restart to ensure to get the latest CPU pages mapping to GPU, then start 1665 * the queues. 1666 */ 1667 static int 1668 svm_range_evict(struct svm_range *prange, struct mm_struct *mm, 1669 unsigned long start, unsigned long last) 1670 { 1671 struct svm_range_list *svms = prange->svms; 1672 struct svm_range *pchild; 1673 struct kfd_process *p; 1674 int r = 0; 1675 1676 p = container_of(svms, struct kfd_process, svms); 1677 1678 pr_debug("invalidate svms 0x%p prange [0x%lx 0x%lx] [0x%lx 0x%lx]\n", 1679 svms, prange->start, prange->last, start, last); 1680 1681 if (!p->xnack_enabled) { 1682 int evicted_ranges; 1683 1684 list_for_each_entry(pchild, &prange->child_list, child_list) { 1685 mutex_lock_nested(&pchild->lock, 1); 1686 if (pchild->start <= last && pchild->last >= start) { 1687 pr_debug("increment pchild invalid [0x%lx 0x%lx]\n", 1688 pchild->start, pchild->last); 1689 atomic_inc(&pchild->invalid); 1690 } 1691 mutex_unlock(&pchild->lock); 1692 } 1693 1694 if (prange->start <= last && prange->last >= start) 1695 atomic_inc(&prange->invalid); 1696 1697 evicted_ranges = atomic_inc_return(&svms->evicted_ranges); 1698 if (evicted_ranges != 1) 1699 return r; 1700 1701 pr_debug("evicting svms 0x%p range [0x%lx 0x%lx]\n", 1702 prange->svms, prange->start, prange->last); 1703 1704 /* First eviction, stop the queues */ 1705 r = kgd2kfd_quiesce_mm(mm); 1706 if (r) 1707 pr_debug("failed to quiesce KFD\n"); 1708 1709 pr_debug("schedule to restore svm %p ranges\n", svms); 1710 schedule_delayed_work(&svms->restore_work, 1711 msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS)); 1712 } else { 1713 unsigned long s, l; 1714 1715 pr_debug("invalidate unmap svms 0x%p [0x%lx 0x%lx] from GPUs\n", 1716 prange->svms, start, last); 1717 list_for_each_entry(pchild, &prange->child_list, child_list) { 1718 mutex_lock_nested(&pchild->lock, 1); 1719 s = max(start, pchild->start); 1720 l = min(last, pchild->last); 1721 if (l >= s) 1722 svm_range_unmap_from_gpus(pchild, s, l); 1723 mutex_unlock(&pchild->lock); 1724 } 1725 s = max(start, prange->start); 1726 l = min(last, prange->last); 1727 if (l >= s) 1728 svm_range_unmap_from_gpus(prange, s, l); 1729 } 1730 1731 return r; 1732 } 1733 1734 static struct svm_range *svm_range_clone(struct svm_range *old) 1735 { 1736 struct svm_range *new; 1737 1738 new = svm_range_new(old->svms, old->start, old->last); 1739 if (!new) 1740 return NULL; 1741 1742 if (old->svm_bo) { 1743 new->ttm_res = old->ttm_res; 1744 new->offset = old->offset; 1745 new->svm_bo = svm_range_bo_ref(old->svm_bo); 1746 spin_lock(&new->svm_bo->list_lock); 1747 list_add(&new->svm_bo_list, &new->svm_bo->range_list); 1748 spin_unlock(&new->svm_bo->list_lock); 1749 } 1750 new->flags = old->flags; 1751 new->preferred_loc = old->preferred_loc; 1752 new->prefetch_loc = old->prefetch_loc; 1753 new->actual_loc = old->actual_loc; 1754 new->granularity = old->granularity; 1755 bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE); 1756 bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE); 1757 1758 return new; 1759 } 1760 1761 /** 1762 * svm_range_handle_overlap - split overlap ranges 1763 * @svms: svm range list header 1764 * @new: range added with this attributes 1765 * @start: range added start address, in pages 1766 * @last: range last address, in pages 1767 * @update_list: output, the ranges attributes are updated. For set_attr, this 1768 * will do validation and map to GPUs. For unmap, this will be 1769 * removed and unmap from GPUs 1770 * @insert_list: output, the ranges will be inserted into svms, attributes are 1771 * not changes. For set_attr, this will add into svms. 1772 * @remove_list:output, the ranges will be removed from svms 1773 * @left: the remaining range after overlap, For set_attr, this will be added 1774 * as new range. 1775 * 1776 * Total have 5 overlap cases. 1777 * 1778 * This function handles overlap of an address interval with existing 1779 * struct svm_ranges for applying new attributes. This may require 1780 * splitting existing struct svm_ranges. All changes should be applied to 1781 * the range_list and interval tree transactionally. If any split operation 1782 * fails, the entire update fails. Therefore the existing overlapping 1783 * svm_ranges are cloned and the original svm_ranges left unchanged. If the 1784 * transaction succeeds, the modified clones are added and the originals 1785 * freed. Otherwise the clones are removed and the old svm_ranges remain. 1786 * 1787 * Context: The caller must hold svms->lock 1788 */ 1789 static int 1790 svm_range_handle_overlap(struct svm_range_list *svms, struct svm_range *new, 1791 unsigned long start, unsigned long last, 1792 struct list_head *update_list, 1793 struct list_head *insert_list, 1794 struct list_head *remove_list, 1795 unsigned long *left) 1796 { 1797 struct interval_tree_node *node; 1798 struct svm_range *prange; 1799 struct svm_range *tmp; 1800 int r = 0; 1801 1802 INIT_LIST_HEAD(update_list); 1803 INIT_LIST_HEAD(insert_list); 1804 INIT_LIST_HEAD(remove_list); 1805 1806 node = interval_tree_iter_first(&svms->objects, start, last); 1807 while (node) { 1808 struct interval_tree_node *next; 1809 struct svm_range *old; 1810 unsigned long next_start; 1811 1812 pr_debug("found overlap node [0x%lx 0x%lx]\n", node->start, 1813 node->last); 1814 1815 old = container_of(node, struct svm_range, it_node); 1816 next = interval_tree_iter_next(node, start, last); 1817 next_start = min(node->last, last) + 1; 1818 1819 if (node->start < start || node->last > last) { 1820 /* node intersects the updated range, clone+split it */ 1821 prange = svm_range_clone(old); 1822 if (!prange) { 1823 r = -ENOMEM; 1824 goto out; 1825 } 1826 1827 list_add(&old->remove_list, remove_list); 1828 list_add(&prange->insert_list, insert_list); 1829 1830 if (node->start < start) { 1831 pr_debug("change old range start\n"); 1832 r = svm_range_split_head(prange, new, start, 1833 insert_list); 1834 if (r) 1835 goto out; 1836 } 1837 if (node->last > last) { 1838 pr_debug("change old range last\n"); 1839 r = svm_range_split_tail(prange, new, last, 1840 insert_list); 1841 if (r) 1842 goto out; 1843 } 1844 } else { 1845 /* The node is contained within start..last, 1846 * just update it 1847 */ 1848 prange = old; 1849 } 1850 1851 if (!svm_range_is_same_attrs(prange, new)) 1852 list_add(&prange->update_list, update_list); 1853 1854 /* insert a new node if needed */ 1855 if (node->start > start) { 1856 prange = svm_range_new(prange->svms, start, 1857 node->start - 1); 1858 if (!prange) { 1859 r = -ENOMEM; 1860 goto out; 1861 } 1862 1863 list_add(&prange->insert_list, insert_list); 1864 list_add(&prange->update_list, update_list); 1865 } 1866 1867 node = next; 1868 start = next_start; 1869 } 1870 1871 if (left && start <= last) 1872 *left = last - start + 1; 1873 1874 out: 1875 if (r) 1876 list_for_each_entry_safe(prange, tmp, insert_list, insert_list) 1877 svm_range_free(prange); 1878 1879 return r; 1880 } 1881 1882 static void 1883 svm_range_update_notifier_and_interval_tree(struct mm_struct *mm, 1884 struct svm_range *prange) 1885 { 1886 unsigned long start; 1887 unsigned long last; 1888 1889 start = prange->notifier.interval_tree.start >> PAGE_SHIFT; 1890 last = prange->notifier.interval_tree.last >> PAGE_SHIFT; 1891 1892 if (prange->start == start && prange->last == last) 1893 return; 1894 1895 pr_debug("up notifier 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n", 1896 prange->svms, prange, start, last, prange->start, 1897 prange->last); 1898 1899 if (start != 0 && last != 0) { 1900 interval_tree_remove(&prange->it_node, &prange->svms->objects); 1901 svm_range_remove_notifier(prange); 1902 } 1903 prange->it_node.start = prange->start; 1904 prange->it_node.last = prange->last; 1905 1906 interval_tree_insert(&prange->it_node, &prange->svms->objects); 1907 svm_range_add_notifier_locked(mm, prange); 1908 } 1909 1910 static void 1911 svm_range_handle_list_op(struct svm_range_list *svms, struct svm_range *prange) 1912 { 1913 struct mm_struct *mm = prange->work_item.mm; 1914 1915 switch (prange->work_item.op) { 1916 case SVM_OP_NULL: 1917 pr_debug("NULL OP 0x%p prange 0x%p [0x%lx 0x%lx]\n", 1918 svms, prange, prange->start, prange->last); 1919 break; 1920 case SVM_OP_UNMAP_RANGE: 1921 pr_debug("remove 0x%p prange 0x%p [0x%lx 0x%lx]\n", 1922 svms, prange, prange->start, prange->last); 1923 svm_range_unlink(prange); 1924 svm_range_remove_notifier(prange); 1925 svm_range_free(prange); 1926 break; 1927 case SVM_OP_UPDATE_RANGE_NOTIFIER: 1928 pr_debug("update notifier 0x%p prange 0x%p [0x%lx 0x%lx]\n", 1929 svms, prange, prange->start, prange->last); 1930 svm_range_update_notifier_and_interval_tree(mm, prange); 1931 break; 1932 case SVM_OP_UPDATE_RANGE_NOTIFIER_AND_MAP: 1933 pr_debug("update and map 0x%p prange 0x%p [0x%lx 0x%lx]\n", 1934 svms, prange, prange->start, prange->last); 1935 svm_range_update_notifier_and_interval_tree(mm, prange); 1936 /* TODO: implement deferred validation and mapping */ 1937 break; 1938 case SVM_OP_ADD_RANGE: 1939 pr_debug("add 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms, prange, 1940 prange->start, prange->last); 1941 svm_range_add_to_svms(prange); 1942 svm_range_add_notifier_locked(mm, prange); 1943 break; 1944 case SVM_OP_ADD_RANGE_AND_MAP: 1945 pr_debug("add and map 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms, 1946 prange, prange->start, prange->last); 1947 svm_range_add_to_svms(prange); 1948 svm_range_add_notifier_locked(mm, prange); 1949 /* TODO: implement deferred validation and mapping */ 1950 break; 1951 default: 1952 WARN_ONCE(1, "Unknown prange 0x%p work op %d\n", prange, 1953 prange->work_item.op); 1954 } 1955 } 1956 1957 static void svm_range_drain_retry_fault(struct svm_range_list *svms) 1958 { 1959 struct kfd_process_device *pdd; 1960 struct amdgpu_device *adev; 1961 struct kfd_process *p; 1962 uint32_t i; 1963 1964 p = container_of(svms, struct kfd_process, svms); 1965 1966 for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) { 1967 pdd = p->pdds[i]; 1968 if (!pdd) 1969 continue; 1970 1971 pr_debug("drain retry fault gpu %d svms %p\n", i, svms); 1972 adev = (struct amdgpu_device *)pdd->dev->kgd; 1973 1974 amdgpu_ih_wait_on_checkpoint_process(adev, &adev->irq.ih1); 1975 pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms); 1976 } 1977 } 1978 1979 static void svm_range_deferred_list_work(struct work_struct *work) 1980 { 1981 struct svm_range_list *svms; 1982 struct svm_range *prange; 1983 struct mm_struct *mm; 1984 1985 svms = container_of(work, struct svm_range_list, deferred_list_work); 1986 pr_debug("enter svms 0x%p\n", svms); 1987 1988 spin_lock(&svms->deferred_list_lock); 1989 while (!list_empty(&svms->deferred_range_list)) { 1990 prange = list_first_entry(&svms->deferred_range_list, 1991 struct svm_range, deferred_list); 1992 spin_unlock(&svms->deferred_list_lock); 1993 pr_debug("prange 0x%p [0x%lx 0x%lx] op %d\n", prange, 1994 prange->start, prange->last, prange->work_item.op); 1995 1996 /* Make sure no stale retry fault coming after range is freed */ 1997 if (prange->work_item.op == SVM_OP_UNMAP_RANGE) 1998 svm_range_drain_retry_fault(prange->svms); 1999 2000 mm = prange->work_item.mm; 2001 mmap_write_lock(mm); 2002 mutex_lock(&svms->lock); 2003 2004 /* Remove from deferred_list must be inside mmap write lock, 2005 * otherwise, svm_range_list_lock_and_flush_work may hold mmap 2006 * write lock, and continue because deferred_list is empty, then 2007 * deferred_list handle is blocked by mmap write lock. 2008 */ 2009 spin_lock(&svms->deferred_list_lock); 2010 list_del_init(&prange->deferred_list); 2011 spin_unlock(&svms->deferred_list_lock); 2012 2013 mutex_lock(&prange->migrate_mutex); 2014 while (!list_empty(&prange->child_list)) { 2015 struct svm_range *pchild; 2016 2017 pchild = list_first_entry(&prange->child_list, 2018 struct svm_range, child_list); 2019 pr_debug("child prange 0x%p op %d\n", pchild, 2020 pchild->work_item.op); 2021 list_del_init(&pchild->child_list); 2022 svm_range_handle_list_op(svms, pchild); 2023 } 2024 mutex_unlock(&prange->migrate_mutex); 2025 2026 svm_range_handle_list_op(svms, prange); 2027 mutex_unlock(&svms->lock); 2028 mmap_write_unlock(mm); 2029 2030 spin_lock(&svms->deferred_list_lock); 2031 } 2032 spin_unlock(&svms->deferred_list_lock); 2033 2034 pr_debug("exit svms 0x%p\n", svms); 2035 } 2036 2037 void 2038 svm_range_add_list_work(struct svm_range_list *svms, struct svm_range *prange, 2039 struct mm_struct *mm, enum svm_work_list_ops op) 2040 { 2041 spin_lock(&svms->deferred_list_lock); 2042 /* if prange is on the deferred list */ 2043 if (!list_empty(&prange->deferred_list)) { 2044 pr_debug("update exist prange 0x%p work op %d\n", prange, op); 2045 WARN_ONCE(prange->work_item.mm != mm, "unmatch mm\n"); 2046 if (op != SVM_OP_NULL && 2047 prange->work_item.op != SVM_OP_UNMAP_RANGE) 2048 prange->work_item.op = op; 2049 } else { 2050 prange->work_item.op = op; 2051 prange->work_item.mm = mm; 2052 list_add_tail(&prange->deferred_list, 2053 &prange->svms->deferred_range_list); 2054 pr_debug("add prange 0x%p [0x%lx 0x%lx] to work list op %d\n", 2055 prange, prange->start, prange->last, op); 2056 } 2057 spin_unlock(&svms->deferred_list_lock); 2058 } 2059 2060 void schedule_deferred_list_work(struct svm_range_list *svms) 2061 { 2062 spin_lock(&svms->deferred_list_lock); 2063 if (!list_empty(&svms->deferred_range_list)) 2064 schedule_work(&svms->deferred_list_work); 2065 spin_unlock(&svms->deferred_list_lock); 2066 } 2067 2068 static void 2069 svm_range_unmap_split(struct mm_struct *mm, struct svm_range *parent, 2070 struct svm_range *prange, unsigned long start, 2071 unsigned long last) 2072 { 2073 struct svm_range *head; 2074 struct svm_range *tail; 2075 2076 if (prange->work_item.op == SVM_OP_UNMAP_RANGE) { 2077 pr_debug("prange 0x%p [0x%lx 0x%lx] is already freed\n", prange, 2078 prange->start, prange->last); 2079 return; 2080 } 2081 if (start > prange->last || last < prange->start) 2082 return; 2083 2084 head = tail = prange; 2085 if (start > prange->start) 2086 svm_range_split(prange, prange->start, start - 1, &tail); 2087 if (last < tail->last) 2088 svm_range_split(tail, last + 1, tail->last, &head); 2089 2090 if (head != prange && tail != prange) { 2091 svm_range_add_child(parent, mm, head, SVM_OP_UNMAP_RANGE); 2092 svm_range_add_child(parent, mm, tail, SVM_OP_ADD_RANGE); 2093 } else if (tail != prange) { 2094 svm_range_add_child(parent, mm, tail, SVM_OP_UNMAP_RANGE); 2095 } else if (head != prange) { 2096 svm_range_add_child(parent, mm, head, SVM_OP_UNMAP_RANGE); 2097 } else if (parent != prange) { 2098 prange->work_item.op = SVM_OP_UNMAP_RANGE; 2099 } 2100 } 2101 2102 static void 2103 svm_range_unmap_from_cpu(struct mm_struct *mm, struct svm_range *prange, 2104 unsigned long start, unsigned long last) 2105 { 2106 struct svm_range_list *svms; 2107 struct svm_range *pchild; 2108 struct kfd_process *p; 2109 unsigned long s, l; 2110 bool unmap_parent; 2111 2112 p = kfd_lookup_process_by_mm(mm); 2113 if (!p) 2114 return; 2115 svms = &p->svms; 2116 2117 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n", svms, 2118 prange, prange->start, prange->last, start, last); 2119 2120 unmap_parent = start <= prange->start && last >= prange->last; 2121 2122 list_for_each_entry(pchild, &prange->child_list, child_list) { 2123 mutex_lock_nested(&pchild->lock, 1); 2124 s = max(start, pchild->start); 2125 l = min(last, pchild->last); 2126 if (l >= s) 2127 svm_range_unmap_from_gpus(pchild, s, l); 2128 svm_range_unmap_split(mm, prange, pchild, start, last); 2129 mutex_unlock(&pchild->lock); 2130 } 2131 s = max(start, prange->start); 2132 l = min(last, prange->last); 2133 if (l >= s) 2134 svm_range_unmap_from_gpus(prange, s, l); 2135 svm_range_unmap_split(mm, prange, prange, start, last); 2136 2137 if (unmap_parent) 2138 svm_range_add_list_work(svms, prange, mm, SVM_OP_UNMAP_RANGE); 2139 else 2140 svm_range_add_list_work(svms, prange, mm, 2141 SVM_OP_UPDATE_RANGE_NOTIFIER); 2142 schedule_deferred_list_work(svms); 2143 2144 kfd_unref_process(p); 2145 } 2146 2147 /** 2148 * svm_range_cpu_invalidate_pagetables - interval notifier callback 2149 * 2150 * If event is MMU_NOTIFY_UNMAP, this is from CPU unmap range, otherwise, it 2151 * is from migration, or CPU page invalidation callback. 2152 * 2153 * For unmap event, unmap range from GPUs, remove prange from svms in a delayed 2154 * work thread, and split prange if only part of prange is unmapped. 2155 * 2156 * For invalidation event, if GPU retry fault is not enabled, evict the queues, 2157 * then schedule svm_range_restore_work to update GPU mapping and resume queues. 2158 * If GPU retry fault is enabled, unmap the svm range from GPU, retry fault will 2159 * update GPU mapping to recover. 2160 * 2161 * Context: mmap lock, notifier_invalidate_start lock are held 2162 * for invalidate event, prange lock is held if this is from migration 2163 */ 2164 static bool 2165 svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni, 2166 const struct mmu_notifier_range *range, 2167 unsigned long cur_seq) 2168 { 2169 struct svm_range *prange; 2170 unsigned long start; 2171 unsigned long last; 2172 2173 if (range->event == MMU_NOTIFY_RELEASE) 2174 return true; 2175 2176 start = mni->interval_tree.start; 2177 last = mni->interval_tree.last; 2178 start = (start > range->start ? start : range->start) >> PAGE_SHIFT; 2179 last = (last < (range->end - 1) ? last : range->end - 1) >> PAGE_SHIFT; 2180 pr_debug("[0x%lx 0x%lx] range[0x%lx 0x%lx] notifier[0x%lx 0x%lx] %d\n", 2181 start, last, range->start >> PAGE_SHIFT, 2182 (range->end - 1) >> PAGE_SHIFT, 2183 mni->interval_tree.start >> PAGE_SHIFT, 2184 mni->interval_tree.last >> PAGE_SHIFT, range->event); 2185 2186 prange = container_of(mni, struct svm_range, notifier); 2187 2188 svm_range_lock(prange); 2189 mmu_interval_set_seq(mni, cur_seq); 2190 2191 switch (range->event) { 2192 case MMU_NOTIFY_UNMAP: 2193 svm_range_unmap_from_cpu(mni->mm, prange, start, last); 2194 break; 2195 default: 2196 svm_range_evict(prange, mni->mm, start, last); 2197 break; 2198 } 2199 2200 svm_range_unlock(prange); 2201 2202 return true; 2203 } 2204 2205 /** 2206 * svm_range_from_addr - find svm range from fault address 2207 * @svms: svm range list header 2208 * @addr: address to search range interval tree, in pages 2209 * @parent: parent range if range is on child list 2210 * 2211 * Context: The caller must hold svms->lock 2212 * 2213 * Return: the svm_range found or NULL 2214 */ 2215 struct svm_range * 2216 svm_range_from_addr(struct svm_range_list *svms, unsigned long addr, 2217 struct svm_range **parent) 2218 { 2219 struct interval_tree_node *node; 2220 struct svm_range *prange; 2221 struct svm_range *pchild; 2222 2223 node = interval_tree_iter_first(&svms->objects, addr, addr); 2224 if (!node) 2225 return NULL; 2226 2227 prange = container_of(node, struct svm_range, it_node); 2228 pr_debug("address 0x%lx prange [0x%lx 0x%lx] node [0x%lx 0x%lx]\n", 2229 addr, prange->start, prange->last, node->start, node->last); 2230 2231 if (addr >= prange->start && addr <= prange->last) { 2232 if (parent) 2233 *parent = prange; 2234 return prange; 2235 } 2236 list_for_each_entry(pchild, &prange->child_list, child_list) 2237 if (addr >= pchild->start && addr <= pchild->last) { 2238 pr_debug("found address 0x%lx pchild [0x%lx 0x%lx]\n", 2239 addr, pchild->start, pchild->last); 2240 if (parent) 2241 *parent = prange; 2242 return pchild; 2243 } 2244 2245 return NULL; 2246 } 2247 2248 /* svm_range_best_restore_location - decide the best fault restore location 2249 * @prange: svm range structure 2250 * @adev: the GPU on which vm fault happened 2251 * 2252 * This is only called when xnack is on, to decide the best location to restore 2253 * the range mapping after GPU vm fault. Caller uses the best location to do 2254 * migration if actual loc is not best location, then update GPU page table 2255 * mapping to the best location. 2256 * 2257 * If vm fault gpu is range preferred loc, the best_loc is preferred loc. 2258 * If vm fault gpu idx is on range ACCESSIBLE bitmap, best_loc is vm fault gpu 2259 * If vm fault gpu idx is on range ACCESSIBLE_IN_PLACE bitmap, then 2260 * if range actual loc is cpu, best_loc is cpu 2261 * if vm fault gpu is on xgmi same hive of range actual loc gpu, best_loc is 2262 * range actual loc. 2263 * Otherwise, GPU no access, best_loc is -1. 2264 * 2265 * Return: 2266 * -1 means vm fault GPU no access 2267 * 0 for CPU or GPU id 2268 */ 2269 static int32_t 2270 svm_range_best_restore_location(struct svm_range *prange, 2271 struct amdgpu_device *adev, 2272 int32_t *gpuidx) 2273 { 2274 struct amdgpu_device *bo_adev; 2275 struct kfd_process *p; 2276 uint32_t gpuid; 2277 int r; 2278 2279 p = container_of(prange->svms, struct kfd_process, svms); 2280 2281 r = kfd_process_gpuid_from_kgd(p, adev, &gpuid, gpuidx); 2282 if (r < 0) { 2283 pr_debug("failed to get gpuid from kgd\n"); 2284 return -1; 2285 } 2286 2287 if (prange->preferred_loc == gpuid) 2288 return prange->preferred_loc; 2289 2290 if (test_bit(*gpuidx, prange->bitmap_access)) 2291 return gpuid; 2292 2293 if (test_bit(*gpuidx, prange->bitmap_aip)) { 2294 if (!prange->actual_loc) 2295 return 0; 2296 2297 bo_adev = svm_range_get_adev_by_id(prange, prange->actual_loc); 2298 if (amdgpu_xgmi_same_hive(adev, bo_adev)) 2299 return prange->actual_loc; 2300 else 2301 return 0; 2302 } 2303 2304 return -1; 2305 } 2306 static int 2307 svm_range_get_range_boundaries(struct kfd_process *p, int64_t addr, 2308 unsigned long *start, unsigned long *last) 2309 { 2310 struct vm_area_struct *vma; 2311 struct interval_tree_node *node; 2312 unsigned long start_limit, end_limit; 2313 2314 vma = find_vma(p->mm, addr << PAGE_SHIFT); 2315 if (!vma || (addr << PAGE_SHIFT) < vma->vm_start) { 2316 pr_debug("VMA does not exist in address [0x%llx]\n", addr); 2317 return -EFAULT; 2318 } 2319 start_limit = max(vma->vm_start >> PAGE_SHIFT, 2320 (unsigned long)ALIGN_DOWN(addr, 2UL << 8)); 2321 end_limit = min(vma->vm_end >> PAGE_SHIFT, 2322 (unsigned long)ALIGN(addr + 1, 2UL << 8)); 2323 /* First range that starts after the fault address */ 2324 node = interval_tree_iter_first(&p->svms.objects, addr + 1, ULONG_MAX); 2325 if (node) { 2326 end_limit = min(end_limit, node->start); 2327 /* Last range that ends before the fault address */ 2328 node = container_of(rb_prev(&node->rb), 2329 struct interval_tree_node, rb); 2330 } else { 2331 /* Last range must end before addr because 2332 * there was no range after addr 2333 */ 2334 node = container_of(rb_last(&p->svms.objects.rb_root), 2335 struct interval_tree_node, rb); 2336 } 2337 if (node) { 2338 if (node->last >= addr) { 2339 WARN(1, "Overlap with prev node and page fault addr\n"); 2340 return -EFAULT; 2341 } 2342 start_limit = max(start_limit, node->last + 1); 2343 } 2344 2345 *start = start_limit; 2346 *last = end_limit - 1; 2347 2348 pr_debug("vma start: 0x%lx start: 0x%lx vma end: 0x%lx last: 0x%lx\n", 2349 vma->vm_start >> PAGE_SHIFT, *start, 2350 vma->vm_end >> PAGE_SHIFT, *last); 2351 2352 return 0; 2353 2354 } 2355 static struct 2356 svm_range *svm_range_create_unregistered_range(struct amdgpu_device *adev, 2357 struct kfd_process *p, 2358 struct mm_struct *mm, 2359 int64_t addr) 2360 { 2361 struct svm_range *prange = NULL; 2362 unsigned long start, last; 2363 uint32_t gpuid, gpuidx; 2364 2365 if (svm_range_get_range_boundaries(p, addr, &start, &last)) 2366 return NULL; 2367 2368 prange = svm_range_new(&p->svms, start, last); 2369 if (!prange) { 2370 pr_debug("Failed to create prange in address [0x%llx]\n", addr); 2371 return NULL; 2372 } 2373 if (kfd_process_gpuid_from_kgd(p, adev, &gpuid, &gpuidx)) { 2374 pr_debug("failed to get gpuid from kgd\n"); 2375 svm_range_free(prange); 2376 return NULL; 2377 } 2378 2379 svm_range_add_to_svms(prange); 2380 svm_range_add_notifier_locked(mm, prange); 2381 2382 return prange; 2383 } 2384 2385 /* svm_range_skip_recover - decide if prange can be recovered 2386 * @prange: svm range structure 2387 * 2388 * GPU vm retry fault handle skip recover the range for cases: 2389 * 1. prange is on deferred list to be removed after unmap, it is stale fault, 2390 * deferred list work will drain the stale fault before free the prange. 2391 * 2. prange is on deferred list to add interval notifier after split, or 2392 * 3. prange is child range, it is split from parent prange, recover later 2393 * after interval notifier is added. 2394 * 2395 * Return: true to skip recover, false to recover 2396 */ 2397 static bool svm_range_skip_recover(struct svm_range *prange) 2398 { 2399 struct svm_range_list *svms = prange->svms; 2400 2401 spin_lock(&svms->deferred_list_lock); 2402 if (list_empty(&prange->deferred_list) && 2403 list_empty(&prange->child_list)) { 2404 spin_unlock(&svms->deferred_list_lock); 2405 return false; 2406 } 2407 spin_unlock(&svms->deferred_list_lock); 2408 2409 if (prange->work_item.op == SVM_OP_UNMAP_RANGE) { 2410 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] unmapped\n", 2411 svms, prange, prange->start, prange->last); 2412 return true; 2413 } 2414 if (prange->work_item.op == SVM_OP_ADD_RANGE_AND_MAP || 2415 prange->work_item.op == SVM_OP_ADD_RANGE) { 2416 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] not added yet\n", 2417 svms, prange, prange->start, prange->last); 2418 return true; 2419 } 2420 return false; 2421 } 2422 2423 static void 2424 svm_range_count_fault(struct amdgpu_device *adev, struct kfd_process *p, 2425 int32_t gpuidx) 2426 { 2427 struct kfd_process_device *pdd; 2428 2429 /* fault is on different page of same range 2430 * or fault is skipped to recover later 2431 * or fault is on invalid virtual address 2432 */ 2433 if (gpuidx == MAX_GPU_INSTANCE) { 2434 uint32_t gpuid; 2435 int r; 2436 2437 r = kfd_process_gpuid_from_kgd(p, adev, &gpuid, &gpuidx); 2438 if (r < 0) 2439 return; 2440 } 2441 2442 /* fault is recovered 2443 * or fault cannot recover because GPU no access on the range 2444 */ 2445 pdd = kfd_process_device_from_gpuidx(p, gpuidx); 2446 if (pdd) 2447 WRITE_ONCE(pdd->faults, pdd->faults + 1); 2448 } 2449 2450 static bool 2451 svm_fault_allowed(struct mm_struct *mm, uint64_t addr, bool write_fault) 2452 { 2453 unsigned long requested = VM_READ; 2454 struct vm_area_struct *vma; 2455 2456 if (write_fault) 2457 requested |= VM_WRITE; 2458 2459 vma = find_vma(mm, addr << PAGE_SHIFT); 2460 if (!vma || (addr << PAGE_SHIFT) < vma->vm_start) { 2461 pr_debug("address 0x%llx VMA is removed\n", addr); 2462 return true; 2463 } 2464 2465 pr_debug("requested 0x%lx, vma permission flags 0x%lx\n", requested, 2466 vma->vm_flags); 2467 return (vma->vm_flags & requested) == requested; 2468 } 2469 2470 int 2471 svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, 2472 uint64_t addr, bool write_fault) 2473 { 2474 struct mm_struct *mm = NULL; 2475 struct svm_range_list *svms; 2476 struct svm_range *prange; 2477 struct kfd_process *p; 2478 uint64_t timestamp; 2479 int32_t best_loc; 2480 int32_t gpuidx = MAX_GPU_INSTANCE; 2481 bool write_locked = false; 2482 int r = 0; 2483 2484 if (!KFD_IS_SVM_API_SUPPORTED(adev->kfd.dev)) { 2485 pr_debug("device does not support SVM\n"); 2486 return -EFAULT; 2487 } 2488 2489 p = kfd_lookup_process_by_pasid(pasid); 2490 if (!p) { 2491 pr_debug("kfd process not founded pasid 0x%x\n", pasid); 2492 return -ESRCH; 2493 } 2494 if (!p->xnack_enabled) { 2495 pr_debug("XNACK not enabled for pasid 0x%x\n", pasid); 2496 r = -EFAULT; 2497 goto out; 2498 } 2499 svms = &p->svms; 2500 2501 pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr); 2502 2503 mm = get_task_mm(p->lead_thread); 2504 if (!mm) { 2505 pr_debug("svms 0x%p failed to get mm\n", svms); 2506 r = -ESRCH; 2507 goto out; 2508 } 2509 2510 mmap_read_lock(mm); 2511 retry_write_locked: 2512 mutex_lock(&svms->lock); 2513 prange = svm_range_from_addr(svms, addr, NULL); 2514 if (!prange) { 2515 pr_debug("failed to find prange svms 0x%p address [0x%llx]\n", 2516 svms, addr); 2517 if (!write_locked) { 2518 /* Need the write lock to create new range with MMU notifier. 2519 * Also flush pending deferred work to make sure the interval 2520 * tree is up to date before we add a new range 2521 */ 2522 mutex_unlock(&svms->lock); 2523 mmap_read_unlock(mm); 2524 mmap_write_lock(mm); 2525 write_locked = true; 2526 goto retry_write_locked; 2527 } 2528 prange = svm_range_create_unregistered_range(adev, p, mm, addr); 2529 if (!prange) { 2530 pr_debug("failed to create unregistered range svms 0x%p address [0x%llx]\n", 2531 svms, addr); 2532 mmap_write_downgrade(mm); 2533 r = -EFAULT; 2534 goto out_unlock_svms; 2535 } 2536 } 2537 if (write_locked) 2538 mmap_write_downgrade(mm); 2539 2540 mutex_lock(&prange->migrate_mutex); 2541 2542 if (svm_range_skip_recover(prange)) { 2543 amdgpu_gmc_filter_faults_remove(adev, addr, pasid); 2544 goto out_unlock_range; 2545 } 2546 2547 timestamp = ktime_to_us(ktime_get()) - prange->validate_timestamp; 2548 /* skip duplicate vm fault on different pages of same range */ 2549 if (timestamp < AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING) { 2550 pr_debug("svms 0x%p [0x%lx %lx] already restored\n", 2551 svms, prange->start, prange->last); 2552 goto out_unlock_range; 2553 } 2554 2555 if (!svm_fault_allowed(mm, addr, write_fault)) { 2556 pr_debug("fault addr 0x%llx no %s permission\n", addr, 2557 write_fault ? "write" : "read"); 2558 r = -EPERM; 2559 goto out_unlock_range; 2560 } 2561 2562 best_loc = svm_range_best_restore_location(prange, adev, &gpuidx); 2563 if (best_loc == -1) { 2564 pr_debug("svms %p failed get best restore loc [0x%lx 0x%lx]\n", 2565 svms, prange->start, prange->last); 2566 r = -EACCES; 2567 goto out_unlock_range; 2568 } 2569 2570 pr_debug("svms %p [0x%lx 0x%lx] best restore 0x%x, actual loc 0x%x\n", 2571 svms, prange->start, prange->last, best_loc, 2572 prange->actual_loc); 2573 2574 if (prange->actual_loc != best_loc) { 2575 if (best_loc) { 2576 r = svm_migrate_to_vram(prange, best_loc, mm); 2577 if (r) { 2578 pr_debug("svm_migrate_to_vram failed (%d) at %llx, falling back to system memory\n", 2579 r, addr); 2580 /* Fallback to system memory if migration to 2581 * VRAM failed 2582 */ 2583 if (prange->actual_loc) 2584 r = svm_migrate_vram_to_ram(prange, mm); 2585 else 2586 r = 0; 2587 } 2588 } else { 2589 r = svm_migrate_vram_to_ram(prange, mm); 2590 } 2591 if (r) { 2592 pr_debug("failed %d to migrate svms %p [0x%lx 0x%lx]\n", 2593 r, svms, prange->start, prange->last); 2594 goto out_unlock_range; 2595 } 2596 } 2597 2598 r = svm_range_validate_and_map(mm, prange, gpuidx, false, false); 2599 if (r) 2600 pr_debug("failed %d to map svms 0x%p [0x%lx 0x%lx] to gpus\n", 2601 r, svms, prange->start, prange->last); 2602 2603 out_unlock_range: 2604 mutex_unlock(&prange->migrate_mutex); 2605 out_unlock_svms: 2606 mutex_unlock(&svms->lock); 2607 mmap_read_unlock(mm); 2608 2609 svm_range_count_fault(adev, p, gpuidx); 2610 2611 mmput(mm); 2612 out: 2613 kfd_unref_process(p); 2614 2615 if (r == -EAGAIN) { 2616 pr_debug("recover vm fault later\n"); 2617 amdgpu_gmc_filter_faults_remove(adev, addr, pasid); 2618 r = 0; 2619 } 2620 return r; 2621 } 2622 2623 void svm_range_list_fini(struct kfd_process *p) 2624 { 2625 struct svm_range *prange; 2626 struct svm_range *next; 2627 2628 pr_debug("pasid 0x%x svms 0x%p\n", p->pasid, &p->svms); 2629 2630 /* Ensure list work is finished before process is destroyed */ 2631 flush_work(&p->svms.deferred_list_work); 2632 2633 list_for_each_entry_safe(prange, next, &p->svms.list, list) { 2634 svm_range_unlink(prange); 2635 svm_range_remove_notifier(prange); 2636 svm_range_free(prange); 2637 } 2638 2639 mutex_destroy(&p->svms.lock); 2640 2641 pr_debug("pasid 0x%x svms 0x%p done\n", p->pasid, &p->svms); 2642 } 2643 2644 int svm_range_list_init(struct kfd_process *p) 2645 { 2646 struct svm_range_list *svms = &p->svms; 2647 int i; 2648 2649 svms->objects = RB_ROOT_CACHED; 2650 mutex_init(&svms->lock); 2651 INIT_LIST_HEAD(&svms->list); 2652 atomic_set(&svms->evicted_ranges, 0); 2653 INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work); 2654 INIT_WORK(&svms->deferred_list_work, svm_range_deferred_list_work); 2655 INIT_LIST_HEAD(&svms->deferred_range_list); 2656 spin_lock_init(&svms->deferred_list_lock); 2657 2658 for (i = 0; i < p->n_pdds; i++) 2659 if (KFD_IS_SVM_API_SUPPORTED(p->pdds[i]->dev)) 2660 bitmap_set(svms->bitmap_supported, i, 1); 2661 2662 return 0; 2663 } 2664 2665 /** 2666 * svm_range_is_valid - check if virtual address range is valid 2667 * @mm: current process mm_struct 2668 * @start: range start address, in pages 2669 * @size: range size, in pages 2670 * 2671 * Valid virtual address range means it belongs to one or more VMAs 2672 * 2673 * Context: Process context 2674 * 2675 * Return: 2676 * true - valid svm range 2677 * false - invalid svm range 2678 */ 2679 static bool 2680 svm_range_is_valid(struct mm_struct *mm, uint64_t start, uint64_t size) 2681 { 2682 const unsigned long device_vma = VM_IO | VM_PFNMAP | VM_MIXEDMAP; 2683 struct vm_area_struct *vma; 2684 unsigned long end; 2685 2686 start <<= PAGE_SHIFT; 2687 end = start + (size << PAGE_SHIFT); 2688 2689 do { 2690 vma = find_vma(mm, start); 2691 if (!vma || start < vma->vm_start || 2692 (vma->vm_flags & device_vma)) 2693 return false; 2694 start = min(end, vma->vm_end); 2695 } while (start < end); 2696 2697 return true; 2698 } 2699 2700 /** 2701 * svm_range_add - add svm range and handle overlap 2702 * @p: the range add to this process svms 2703 * @start: page size aligned 2704 * @size: page size aligned 2705 * @nattr: number of attributes 2706 * @attrs: array of attributes 2707 * @update_list: output, the ranges need validate and update GPU mapping 2708 * @insert_list: output, the ranges need insert to svms 2709 * @remove_list: output, the ranges are replaced and need remove from svms 2710 * 2711 * Check if the virtual address range has overlap with the registered ranges, 2712 * split the overlapped range, copy and adjust pages address and vram nodes in 2713 * old and new ranges. 2714 * 2715 * Context: Process context, caller must hold svms->lock 2716 * 2717 * Return: 2718 * 0 - OK, otherwise error code 2719 */ 2720 static int 2721 svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size, 2722 uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs, 2723 struct list_head *update_list, struct list_head *insert_list, 2724 struct list_head *remove_list) 2725 { 2726 uint64_t last = start + size - 1UL; 2727 struct svm_range_list *svms; 2728 struct svm_range new = {0}; 2729 struct svm_range *prange; 2730 unsigned long left = 0; 2731 int r = 0; 2732 2733 pr_debug("svms 0x%p [0x%llx 0x%llx]\n", &p->svms, start, last); 2734 2735 svm_range_apply_attrs(p, &new, nattr, attrs); 2736 2737 svms = &p->svms; 2738 2739 r = svm_range_handle_overlap(svms, &new, start, last, update_list, 2740 insert_list, remove_list, &left); 2741 if (r) 2742 return r; 2743 2744 if (left) { 2745 prange = svm_range_new(svms, last - left + 1, last); 2746 list_add(&prange->insert_list, insert_list); 2747 list_add(&prange->update_list, update_list); 2748 } 2749 2750 return 0; 2751 } 2752 2753 /** 2754 * svm_range_best_prefetch_location - decide the best prefetch location 2755 * @prange: svm range structure 2756 * 2757 * For xnack off: 2758 * If range map to single GPU, the best prefetch location is prefetch_loc, which 2759 * can be CPU or GPU. 2760 * 2761 * If range is ACCESS or ACCESS_IN_PLACE by mGPUs, only if mGPU connection on 2762 * XGMI same hive, the best prefetch location is prefetch_loc GPU, othervise 2763 * the best prefetch location is always CPU, because GPU can not have coherent 2764 * mapping VRAM of other GPUs even with large-BAR PCIe connection. 2765 * 2766 * For xnack on: 2767 * If range is not ACCESS_IN_PLACE by mGPUs, the best prefetch location is 2768 * prefetch_loc, other GPU access will generate vm fault and trigger migration. 2769 * 2770 * If range is ACCESS_IN_PLACE by mGPUs, only if mGPU connection on XGMI same 2771 * hive, the best prefetch location is prefetch_loc GPU, otherwise the best 2772 * prefetch location is always CPU. 2773 * 2774 * Context: Process context 2775 * 2776 * Return: 2777 * 0 for CPU or GPU id 2778 */ 2779 static uint32_t 2780 svm_range_best_prefetch_location(struct svm_range *prange) 2781 { 2782 DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE); 2783 uint32_t best_loc = prange->prefetch_loc; 2784 struct kfd_process_device *pdd; 2785 struct amdgpu_device *bo_adev; 2786 struct amdgpu_device *adev; 2787 struct kfd_process *p; 2788 uint32_t gpuidx; 2789 2790 p = container_of(prange->svms, struct kfd_process, svms); 2791 2792 if (!best_loc || best_loc == KFD_IOCTL_SVM_LOCATION_UNDEFINED) 2793 goto out; 2794 2795 bo_adev = svm_range_get_adev_by_id(prange, best_loc); 2796 if (!bo_adev) { 2797 WARN_ONCE(1, "failed to get device by id 0x%x\n", best_loc); 2798 best_loc = 0; 2799 goto out; 2800 } 2801 2802 if (p->xnack_enabled) 2803 bitmap_copy(bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE); 2804 else 2805 bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip, 2806 MAX_GPU_INSTANCE); 2807 2808 for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) { 2809 pdd = kfd_process_device_from_gpuidx(p, gpuidx); 2810 if (!pdd) { 2811 pr_debug("failed to get device by idx 0x%x\n", gpuidx); 2812 continue; 2813 } 2814 adev = (struct amdgpu_device *)pdd->dev->kgd; 2815 2816 if (adev == bo_adev) 2817 continue; 2818 2819 if (!amdgpu_xgmi_same_hive(adev, bo_adev)) { 2820 best_loc = 0; 2821 break; 2822 } 2823 } 2824 2825 out: 2826 pr_debug("xnack %d svms 0x%p [0x%lx 0x%lx] best loc 0x%x\n", 2827 p->xnack_enabled, &p->svms, prange->start, prange->last, 2828 best_loc); 2829 2830 return best_loc; 2831 } 2832 2833 /* FIXME: This is a workaround for page locking bug when some pages are 2834 * invalid during migration to VRAM 2835 */ 2836 void svm_range_prefault(struct svm_range *prange, struct mm_struct *mm, 2837 void *owner) 2838 { 2839 struct hmm_range *hmm_range; 2840 int r; 2841 2842 if (prange->validated_once) 2843 return; 2844 2845 r = amdgpu_hmm_range_get_pages(&prange->notifier, mm, NULL, 2846 prange->start << PAGE_SHIFT, 2847 prange->npages, &hmm_range, 2848 false, true, owner); 2849 if (!r) { 2850 amdgpu_hmm_range_get_pages_done(hmm_range); 2851 prange->validated_once = true; 2852 } 2853 } 2854 2855 /* svm_range_trigger_migration - start page migration if prefetch loc changed 2856 * @mm: current process mm_struct 2857 * @prange: svm range structure 2858 * @migrated: output, true if migration is triggered 2859 * 2860 * If range perfetch_loc is GPU, actual loc is cpu 0, then migrate the range 2861 * from ram to vram. 2862 * If range prefetch_loc is cpu 0, actual loc is GPU, then migrate the range 2863 * from vram to ram. 2864 * 2865 * If GPU vm fault retry is not enabled, migration interact with MMU notifier 2866 * and restore work: 2867 * 1. migrate_vma_setup invalidate pages, MMU notifier callback svm_range_evict 2868 * stops all queues, schedule restore work 2869 * 2. svm_range_restore_work wait for migration is done by 2870 * a. svm_range_validate_vram takes prange->migrate_mutex 2871 * b. svm_range_validate_ram HMM get pages wait for CPU fault handle returns 2872 * 3. restore work update mappings of GPU, resume all queues. 2873 * 2874 * Context: Process context 2875 * 2876 * Return: 2877 * 0 - OK, otherwise - error code of migration 2878 */ 2879 static int 2880 svm_range_trigger_migration(struct mm_struct *mm, struct svm_range *prange, 2881 bool *migrated) 2882 { 2883 uint32_t best_loc; 2884 int r = 0; 2885 2886 *migrated = false; 2887 best_loc = svm_range_best_prefetch_location(prange); 2888 2889 if (best_loc == KFD_IOCTL_SVM_LOCATION_UNDEFINED || 2890 best_loc == prange->actual_loc) 2891 return 0; 2892 2893 if (!best_loc) { 2894 r = svm_migrate_vram_to_ram(prange, mm); 2895 *migrated = !r; 2896 return r; 2897 } 2898 2899 r = svm_migrate_to_vram(prange, best_loc, mm); 2900 *migrated = !r; 2901 2902 return r; 2903 } 2904 2905 int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence) 2906 { 2907 if (!fence) 2908 return -EINVAL; 2909 2910 if (dma_fence_is_signaled(&fence->base)) 2911 return 0; 2912 2913 if (fence->svm_bo) { 2914 WRITE_ONCE(fence->svm_bo->evicting, 1); 2915 schedule_work(&fence->svm_bo->eviction_work); 2916 } 2917 2918 return 0; 2919 } 2920 2921 static void svm_range_evict_svm_bo_worker(struct work_struct *work) 2922 { 2923 struct svm_range_bo *svm_bo; 2924 struct kfd_process *p; 2925 struct mm_struct *mm; 2926 2927 svm_bo = container_of(work, struct svm_range_bo, eviction_work); 2928 if (!svm_bo_ref_unless_zero(svm_bo)) 2929 return; /* svm_bo was freed while eviction was pending */ 2930 2931 /* svm_range_bo_release destroys this worker thread. So during 2932 * the lifetime of this thread, kfd_process and mm will be valid. 2933 */ 2934 p = container_of(svm_bo->svms, struct kfd_process, svms); 2935 mm = p->mm; 2936 if (!mm) 2937 return; 2938 2939 mmap_read_lock(mm); 2940 spin_lock(&svm_bo->list_lock); 2941 while (!list_empty(&svm_bo->range_list)) { 2942 struct svm_range *prange = 2943 list_first_entry(&svm_bo->range_list, 2944 struct svm_range, svm_bo_list); 2945 list_del_init(&prange->svm_bo_list); 2946 spin_unlock(&svm_bo->list_lock); 2947 2948 pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, 2949 prange->start, prange->last); 2950 2951 mutex_lock(&prange->migrate_mutex); 2952 svm_migrate_vram_to_ram(prange, svm_bo->eviction_fence->mm); 2953 2954 mutex_lock(&prange->lock); 2955 prange->svm_bo = NULL; 2956 mutex_unlock(&prange->lock); 2957 2958 mutex_unlock(&prange->migrate_mutex); 2959 2960 spin_lock(&svm_bo->list_lock); 2961 } 2962 spin_unlock(&svm_bo->list_lock); 2963 mmap_read_unlock(mm); 2964 2965 dma_fence_signal(&svm_bo->eviction_fence->base); 2966 /* This is the last reference to svm_bo, after svm_range_vram_node_free 2967 * has been called in svm_migrate_vram_to_ram 2968 */ 2969 WARN_ONCE(kref_read(&svm_bo->kref) != 1, "This was not the last reference\n"); 2970 svm_range_bo_unref(svm_bo); 2971 } 2972 2973 static int 2974 svm_range_set_attr(struct kfd_process *p, uint64_t start, uint64_t size, 2975 uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs) 2976 { 2977 struct amdkfd_process_info *process_info = p->kgd_process_info; 2978 struct mm_struct *mm = current->mm; 2979 struct list_head update_list; 2980 struct list_head insert_list; 2981 struct list_head remove_list; 2982 struct svm_range_list *svms; 2983 struct svm_range *prange; 2984 struct svm_range *next; 2985 int r = 0; 2986 2987 pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] pages 0x%llx\n", 2988 p->pasid, &p->svms, start, start + size - 1, size); 2989 2990 r = svm_range_check_attr(p, nattr, attrs); 2991 if (r) 2992 return r; 2993 2994 svms = &p->svms; 2995 2996 mutex_lock(&process_info->lock); 2997 2998 svm_range_list_lock_and_flush_work(svms, mm); 2999 3000 if (!svm_range_is_valid(mm, start, size)) { 3001 pr_debug("invalid range\n"); 3002 r = -EFAULT; 3003 mmap_write_unlock(mm); 3004 goto out; 3005 } 3006 3007 mutex_lock(&svms->lock); 3008 3009 /* Add new range and split existing ranges as needed */ 3010 r = svm_range_add(p, start, size, nattr, attrs, &update_list, 3011 &insert_list, &remove_list); 3012 if (r) { 3013 mutex_unlock(&svms->lock); 3014 mmap_write_unlock(mm); 3015 goto out; 3016 } 3017 /* Apply changes as a transaction */ 3018 list_for_each_entry_safe(prange, next, &insert_list, insert_list) { 3019 svm_range_add_to_svms(prange); 3020 svm_range_add_notifier_locked(mm, prange); 3021 } 3022 list_for_each_entry(prange, &update_list, update_list) { 3023 svm_range_apply_attrs(p, prange, nattr, attrs); 3024 /* TODO: unmap ranges from GPU that lost access */ 3025 } 3026 list_for_each_entry_safe(prange, next, &remove_list, 3027 remove_list) { 3028 pr_debug("unlink old 0x%p prange 0x%p [0x%lx 0x%lx]\n", 3029 prange->svms, prange, prange->start, 3030 prange->last); 3031 svm_range_unlink(prange); 3032 svm_range_remove_notifier(prange); 3033 svm_range_free(prange); 3034 } 3035 3036 mmap_write_downgrade(mm); 3037 /* Trigger migrations and revalidate and map to GPUs as needed. If 3038 * this fails we may be left with partially completed actions. There 3039 * is no clean way of rolling back to the previous state in such a 3040 * case because the rollback wouldn't be guaranteed to work either. 3041 */ 3042 list_for_each_entry(prange, &update_list, update_list) { 3043 bool migrated; 3044 3045 mutex_lock(&prange->migrate_mutex); 3046 3047 r = svm_range_trigger_migration(mm, prange, &migrated); 3048 if (r) 3049 goto out_unlock_range; 3050 3051 if (migrated && !p->xnack_enabled) { 3052 pr_debug("restore_work will update mappings of GPUs\n"); 3053 mutex_unlock(&prange->migrate_mutex); 3054 continue; 3055 } 3056 3057 r = svm_range_validate_and_map(mm, prange, MAX_GPU_INSTANCE, 3058 true, true); 3059 if (r) 3060 pr_debug("failed %d to map svm range\n", r); 3061 3062 out_unlock_range: 3063 mutex_unlock(&prange->migrate_mutex); 3064 if (r) 3065 break; 3066 } 3067 3068 svm_range_debug_dump(svms); 3069 3070 mutex_unlock(&svms->lock); 3071 mmap_read_unlock(mm); 3072 out: 3073 mutex_unlock(&process_info->lock); 3074 3075 pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] done, r=%d\n", p->pasid, 3076 &p->svms, start, start + size - 1, r); 3077 3078 return r; 3079 } 3080 3081 static int 3082 svm_range_get_attr(struct kfd_process *p, uint64_t start, uint64_t size, 3083 uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs) 3084 { 3085 DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE); 3086 DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE); 3087 bool get_preferred_loc = false; 3088 bool get_prefetch_loc = false; 3089 bool get_granularity = false; 3090 bool get_accessible = false; 3091 bool get_flags = false; 3092 uint64_t last = start + size - 1UL; 3093 struct mm_struct *mm = current->mm; 3094 uint8_t granularity = 0xff; 3095 struct interval_tree_node *node; 3096 struct svm_range_list *svms; 3097 struct svm_range *prange; 3098 uint32_t prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED; 3099 uint32_t location = KFD_IOCTL_SVM_LOCATION_UNDEFINED; 3100 uint32_t flags_and = 0xffffffff; 3101 uint32_t flags_or = 0; 3102 int gpuidx; 3103 uint32_t i; 3104 3105 pr_debug("svms 0x%p [0x%llx 0x%llx] nattr 0x%x\n", &p->svms, start, 3106 start + size - 1, nattr); 3107 3108 /* Flush pending deferred work to avoid racing with deferred actions from 3109 * previous memory map changes (e.g. munmap). Concurrent memory map changes 3110 * can still race with get_attr because we don't hold the mmap lock. But that 3111 * would be a race condition in the application anyway, and undefined 3112 * behaviour is acceptable in that case. 3113 */ 3114 flush_work(&p->svms.deferred_list_work); 3115 3116 mmap_read_lock(mm); 3117 if (!svm_range_is_valid(mm, start, size)) { 3118 pr_debug("invalid range\n"); 3119 mmap_read_unlock(mm); 3120 return -EINVAL; 3121 } 3122 mmap_read_unlock(mm); 3123 3124 for (i = 0; i < nattr; i++) { 3125 switch (attrs[i].type) { 3126 case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: 3127 get_preferred_loc = true; 3128 break; 3129 case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: 3130 get_prefetch_loc = true; 3131 break; 3132 case KFD_IOCTL_SVM_ATTR_ACCESS: 3133 get_accessible = true; 3134 break; 3135 case KFD_IOCTL_SVM_ATTR_SET_FLAGS: 3136 case KFD_IOCTL_SVM_ATTR_CLR_FLAGS: 3137 get_flags = true; 3138 break; 3139 case KFD_IOCTL_SVM_ATTR_GRANULARITY: 3140 get_granularity = true; 3141 break; 3142 case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE: 3143 case KFD_IOCTL_SVM_ATTR_NO_ACCESS: 3144 fallthrough; 3145 default: 3146 pr_debug("get invalid attr type 0x%x\n", attrs[i].type); 3147 return -EINVAL; 3148 } 3149 } 3150 3151 svms = &p->svms; 3152 3153 mutex_lock(&svms->lock); 3154 3155 node = interval_tree_iter_first(&svms->objects, start, last); 3156 if (!node) { 3157 pr_debug("range attrs not found return default values\n"); 3158 svm_range_set_default_attributes(&location, &prefetch_loc, 3159 &granularity, &flags_and); 3160 flags_or = flags_and; 3161 if (p->xnack_enabled) 3162 bitmap_copy(bitmap_access, svms->bitmap_supported, 3163 MAX_GPU_INSTANCE); 3164 else 3165 bitmap_zero(bitmap_access, MAX_GPU_INSTANCE); 3166 bitmap_zero(bitmap_aip, MAX_GPU_INSTANCE); 3167 goto fill_values; 3168 } 3169 bitmap_copy(bitmap_access, svms->bitmap_supported, MAX_GPU_INSTANCE); 3170 bitmap_copy(bitmap_aip, svms->bitmap_supported, MAX_GPU_INSTANCE); 3171 3172 while (node) { 3173 struct interval_tree_node *next; 3174 3175 prange = container_of(node, struct svm_range, it_node); 3176 next = interval_tree_iter_next(node, start, last); 3177 3178 if (get_preferred_loc) { 3179 if (prange->preferred_loc == 3180 KFD_IOCTL_SVM_LOCATION_UNDEFINED || 3181 (location != KFD_IOCTL_SVM_LOCATION_UNDEFINED && 3182 location != prange->preferred_loc)) { 3183 location = KFD_IOCTL_SVM_LOCATION_UNDEFINED; 3184 get_preferred_loc = false; 3185 } else { 3186 location = prange->preferred_loc; 3187 } 3188 } 3189 if (get_prefetch_loc) { 3190 if (prange->prefetch_loc == 3191 KFD_IOCTL_SVM_LOCATION_UNDEFINED || 3192 (prefetch_loc != KFD_IOCTL_SVM_LOCATION_UNDEFINED && 3193 prefetch_loc != prange->prefetch_loc)) { 3194 prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED; 3195 get_prefetch_loc = false; 3196 } else { 3197 prefetch_loc = prange->prefetch_loc; 3198 } 3199 } 3200 if (get_accessible) { 3201 bitmap_and(bitmap_access, bitmap_access, 3202 prange->bitmap_access, MAX_GPU_INSTANCE); 3203 bitmap_and(bitmap_aip, bitmap_aip, 3204 prange->bitmap_aip, MAX_GPU_INSTANCE); 3205 } 3206 if (get_flags) { 3207 flags_and &= prange->flags; 3208 flags_or |= prange->flags; 3209 } 3210 3211 if (get_granularity && prange->granularity < granularity) 3212 granularity = prange->granularity; 3213 3214 node = next; 3215 } 3216 fill_values: 3217 mutex_unlock(&svms->lock); 3218 3219 for (i = 0; i < nattr; i++) { 3220 switch (attrs[i].type) { 3221 case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: 3222 attrs[i].value = location; 3223 break; 3224 case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: 3225 attrs[i].value = prefetch_loc; 3226 break; 3227 case KFD_IOCTL_SVM_ATTR_ACCESS: 3228 gpuidx = kfd_process_gpuidx_from_gpuid(p, 3229 attrs[i].value); 3230 if (gpuidx < 0) { 3231 pr_debug("invalid gpuid %x\n", attrs[i].value); 3232 return -EINVAL; 3233 } 3234 if (test_bit(gpuidx, bitmap_access)) 3235 attrs[i].type = KFD_IOCTL_SVM_ATTR_ACCESS; 3236 else if (test_bit(gpuidx, bitmap_aip)) 3237 attrs[i].type = 3238 KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE; 3239 else 3240 attrs[i].type = KFD_IOCTL_SVM_ATTR_NO_ACCESS; 3241 break; 3242 case KFD_IOCTL_SVM_ATTR_SET_FLAGS: 3243 attrs[i].value = flags_and; 3244 break; 3245 case KFD_IOCTL_SVM_ATTR_CLR_FLAGS: 3246 attrs[i].value = ~flags_or; 3247 break; 3248 case KFD_IOCTL_SVM_ATTR_GRANULARITY: 3249 attrs[i].value = (uint32_t)granularity; 3250 break; 3251 } 3252 } 3253 3254 return 0; 3255 } 3256 3257 int 3258 svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start, 3259 uint64_t size, uint32_t nattrs, struct kfd_ioctl_svm_attribute *attrs) 3260 { 3261 int r; 3262 3263 start >>= PAGE_SHIFT; 3264 size >>= PAGE_SHIFT; 3265 3266 switch (op) { 3267 case KFD_IOCTL_SVM_OP_SET_ATTR: 3268 r = svm_range_set_attr(p, start, size, nattrs, attrs); 3269 break; 3270 case KFD_IOCTL_SVM_OP_GET_ATTR: 3271 r = svm_range_get_attr(p, start, size, nattrs, attrs); 3272 break; 3273 default: 3274 r = EINVAL; 3275 break; 3276 } 3277 3278 return r; 3279 } 3280